You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tubemq.apache.org by yu...@apache.org on 2021/01/29 10:06:58 UTC
[incubator-tubemq] branch TUBEMQ-421 updated (3acac1a -> e69631b)
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a change to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git.
from 3acac1a [TUBEMQ-531] delete cluster (#407)
new 45f70fc [TUBEMQ-433] add tubemq perf-consumer/producer scripts (#330)
new df95aac [TUBEMQ-440] Add feature package tube-manager to zip (#337)
new 9598a42 [TUBEMQ-430]Optimizing the implementation of HTTP API for broker (#338)
new b2177f1 [TUBEMQ-441]An error occurred when using the Tubemq class to create a sink table (#339)
new a835a50 [TUBEMQ-442]Modifying the jvm parameters when the broker starts does not take effect (#340)
new ea901b7 [TUBEMQ-428] Bumped version to 0.8.0-SNAPSHOT (#341)
new 86ccc70 [TUBEMQ-437] Fix tubemq table source sink factory instance creating problem (#342)
new aec25b5 [TUBEMQ-443] TubemqSourceFunction class prints too many logs problem (#344)
new 6ce60a8 [TUBEMQ-444]Add consume and produce Cli commands (#343)
new 7a1d7cb [TUBEMQ-445]Adjust the status check default sleep interval of pullConsumeReadyChkSliceMs (#345)
new 40a3491 [TUBEMQ-446]Small bugs fix that do not affect the main logics (#346)
new 9565483 [TUBEMQ-447] Add Broker-Admin Cli (#347)
new 8acad06 [TUBEMQ-449]Adjust Example implementation (#348)
new 3b6d610 [TUBEMQ-453] TubemqSourceFunction class prints too many logs problem (#350)
new ea22d0b [TUBEMQ-450]TubeClientException: Generate producer id failed (#351)
new bd132b1 [TUBEMQ-457] There is no need to return StringBuilder in Master.java (#352)
new 9e184b1 [TUBEMQ-451]Replace ConsumeTupleInfo with Tuple2 (#349)
new 380436e [TUBEMQ-463]Adjust Master rebalance process implementation (#355)
new 8e0b01d [TUBEMQ-470] Add query API of TopicName and BrokerId collection
new 4920c51 [TUBEMQ-472]Adjust Broker's AbstractWebHandler class implementation
new cc5796a [TUBEMQ-475] add the offset clone api of the consume group
new d8580f2 [TUBEMQ-482] Add offset query api
new 868b04c [TUBEMQ-484]Add query API for topic publication information
new 1b854fb [TUBEMQ-485]Add the batch setting API of consume group offset
new e3d818c [TUBEMQ-486]Add the delete API of consumer group offset
new 7dd7268 [TUBEMQ-486]Add the delete API of consumer group offset
new e8bd45a [TUBEMQ-495]Code implementation adjustment based on SpotBugs check
new f5ae304 [TUBEMQ-499] Add configure store
new 8762bd1 [TUBEMQ-504]Adjust the WebMethodMapper class interfaces (#388)
new 20c4602 [TUBEMQ-500] Add setting operate API (#389)
new 78ced32 [TUBEMQ-505] Remove the "WIP" label of the DISCLAIMER file (#390)
new 6ca161c [TUBEMQ-501] Adjust max message size check logic
new fb22584 [TUBEMQ-508] Optimize Broker's PB parameter check processing logic (#392)
new 1b7ca30 [TUBEMQ-509] Adjust the packet length check when data is loaded
new d25d00c [TUBEMQ-509] Adjust the packet length check when data is loaded
new a7ae74e [TUBEMQ-510] Found a bug in MessageProducerExample class
new eaf0f10 rm -Werror
new 72213f2 add change
new 78e3685 [TUBEMQ-511]Replace the conditional operator (?:) with mid()
new 5e4129e [TUBEMQ-512] Add package length control based on Topic
new a343a3f [TUBEMQ-518] fix parameter pass error
new 4b138de [TUBEMQ-515]Add cluster Topic view web api
new 0130748 [TUBEMQ-517] Add 0.8.0 version release modification to CHANGES.md (#399)
new 2b34005 [TUBEMQ-526] Adjust the packaging script and version check list, remove the "-WIP" tag
new 3fcbb38 [TUBEMQ-529] Update CHANGE.md
new f1bcd20 [TUBEMQ-544]Adjust the LICENSE statement in the client.conf files of Python and C/C++ SDK
new 9530cae [TUBEMQ-546]Restore the original license header of the referenced external source files
new 56cb40b [TUBEMQ-551] Adjust NOTICE file content
new e69631b [TUBEMQ-547]Recode the implementation of the *Startup.java classes in the Tool package
The 49 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.md | 68 ++
DISCLAIMER | 6 +
DISCLAIMER-WIP | 18 -
LICENSE | 1 -
NOTICE | 155 ++-
bin/env.sh | 2 +-
bin/{groupAdmin.sh => tubemq-broker-admin.sh} | 5 +-
bin/{groupAdmin.sh => tubemq-consumer-test.sh} | 5 +-
bin/{groupAdmin.sh => tubemq-producer-test.sh} | 5 +-
pom.xml | 26 +-
resources/assets/scripts/topicList.js | 8 +-
.../tubemq-client-cpp/CMakeLists.txt | 2 +-
.../tubemq-client-cpp/conf/client.conf | 22 +-
tubemq-client-twins/tubemq-client-cpp/src/any.h | 21 +-
tubemq-client-twins/tubemq-client-cpp/src/buffer.h | 22 +-
.../src/python/tubemq/client.conf | 22 +-
tubemq-client/pom.xml | 2 +-
tubemq-client/src/main/assembly/assembly.xml | 2 +-
.../tubemq/client/common/TClientConstants.java | 2 +-
.../tubemq/client/config/ConsumerConfig.java | 12 +
.../tubemq/client/config/TubeClientConfig.java | 9 +-
.../client/consumer/MessageFetchManager.java | 5 +-
.../client/consumer/PullMessageConsumer.java | 7 +
.../tubemq/client/consumer/RmtDataCache.java | 4 +-
.../client/consumer/SimplePullMessageConsumer.java | 4 +-
.../tubemq/client/producer/AllowedSetting.java | 61 ++
.../tubemq/client/producer/ProducerManager.java | 48 +-
.../client/producer/SimpleMessageProducer.java | 25 +-
tubemq-connectors/pom.xml | 2 +-
tubemq-connectors/tubemq-connector-flink/pom.xml | 2 +-
.../org/apache/flink/connectors/tubemq/Tubemq.java | 28 +-
.../connectors/tubemq/TubemqSourceFunction.java | 19 +-
.../tubemq/TubemqTableSourceSinkFactory.java | 2 +-
.../org.apache.flink.table.factories.TableFactory | 2 +-
tubemq-connectors/tubemq-connector-flume/pom.xml | 2 +-
tubemq-connectors/tubemq-connector-spark/pom.xml | 2 +-
tubemq-core/pom.xml | 2 +-
.../org/apache/tubemq/corebase/TBaseConstants.java | 17 +-
.../apache/tubemq/corebase/TErrCodeConstants.java | 6 +
.../apache/tubemq/corebase/utils/AddressUtils.java | 107 +-
.../apache/tubemq/corebase/utils/MixedUtils.java | 69 ++
.../org/apache/tubemq/corebase/utils/RegexDef.java | 60 ++
...ncurrentHashSet.java => SettingValidUtils.java} | 84 +-
.../apache/tubemq/corebase/utils/TStringUtils.java | 27 +-
.../org/apache/tubemq/corebase/utils/Tuple2.java | 114 ++-
.../utils/{ConcurrentHashSet.java => Tuple3.java} | 106 +-
.../tubemq/corerpc/AbstractServiceInvoker.java | 2 +-
.../org/apache/tubemq/corerpc/RpcConstants.java | 6 +-
.../apache/tubemq/corerpc/RpcServiceFactory.java | 4 +-
tubemq-core/src/main/proto/MasterService.proto | 18 +
tubemq-docker/pom.xml | 2 +-
tubemq-docker/tubemq-all/pom.xml | 2 +-
tubemq-docker/tubemq-build/pom.xml | 2 +-
tubemq-example/pom.xml | 2 +-
tubemq-example/src/main/assembly/assembly.xml | 2 +-
.../tubemq/example/MAMessageProducerExample.java | 158 +--
.../tubemq/example/MessageConsumerExample.java | 60 +-
.../tubemq/example/MessageProducerExample.java | 131 ++-
.../tubemq/example/MessagePullConsumerExample.java | 63 +-
.../example/MessagePullSetConsumerExample.java | 62 +-
tubemq-manager/conf/logback.xml | 64 ++
tubemq-manager/pom.xml | 31 +-
tubemq-manager/src/main/assembly/assembly.xml | 61 ++
tubemq-server/pom.xml | 8 +-
tubemq-server/src/main/assembly/assembly.xml | 2 +-
.../tubemq/server/broker/BrokerServiceServer.java | 191 ++--
.../apache/tubemq/server/broker/TubeBroker.java | 270 +++--
.../broker/metadata/BrokerMetadataManager.java | 3 +-
.../broker/metadata/ClusterConfigHolder.java | 81 ++
.../server/broker/metadata/MetadataManager.java | 2 +
.../server/broker/metadata/TopicMetadata.java | 88 +-
.../server/broker/msgstore/MessageStore.java | 34 +-
.../broker/msgstore/MessageStoreManager.java | 62 +-
.../server/broker/msgstore/StoreService.java | 6 +
.../server/broker/msgstore/mem/MsgMemStore.java | 3 +-
.../server/broker/offset/DefaultOffsetManager.java | 288 +++++-
.../tubemq/server/broker/offset/OffsetService.java | 25 +
.../server/broker/stats/GroupCountService.java | 2 +-
.../tubemq/server/broker/utils/DataStoreUtils.java | 7 +-
.../server/broker/utils/GroupOffsetInfo.java | 85 ++
.../server/broker/utils/TopicPubStoreInfo.java | 55 +
.../server/broker/web/AbstractWebHandler.java | 92 ++
.../server/broker/web/BrokerAdminServlet.java | 1050 ++++++++++++++++----
.../tubemq/server/common/TServerConstants.java | 3 +
.../tubemq/server/common/fielddef/CliArgDef.java | 131 +++
.../tubemq/server/common/fielddef/WebFieldDef.java | 205 ++++
.../tubemq/server/common/fileconfig/ZKConfig.java | 1 -
.../server/common/offsetstorage/OffsetStorage.java | 14 +-
.../common/offsetstorage/ZkOffsetStorage.java | 194 +++-
.../common/offsetstorage/zookeeper/ZKUtil.java | 33 +
.../server/common/paramcheck/PBParameterUtils.java | 193 ++--
.../tubemq/server/common/utils/HttpUtils.java | 112 +++
.../ProcessResult.java} | 122 +--
.../server/common/utils/WebParameterUtils.java | 553 ++++++++++-
.../tubemq/server/common/webbase/WebFieldType.java | 130 ++-
.../webbase/WebMethodMapper.java} | 53 +-
.../apache/tubemq/server/master/MasterConfig.java | 11 +
.../org/apache/tubemq/server/master/TMaster.java | 280 ++++--
.../server/master/bdbstore/BdbStoreService.java | 7 +
.../master/bdbstore/DefaultBdbStoreService.java | 99 +-
.../bdbentitys/BdbClusterSettingEntity.java | 340 +++++++
.../bdbstore/bdbentitys/BdbTopicConfEntity.java | 18 +
.../nodemanage/nodebroker/BrokerConfManager.java | 134 ++-
.../nodemanage/nodebroker/BrokerInfoHolder.java | 4 +-
.../nodebroker/BrokerSyncStatusInfo.java | 15 +
.../nodeconsumer/ConsumerInfoHolder.java | 14 +-
.../server/master/web/action/screen/Master.java | 32 +-
.../server/master/web/action/screen/Webapi.java | 12 +-
.../master/web/handler/AbstractWebHandler.java | 16 +-
.../web/handler/WebBrokerTopicConfHandler.java | 179 +++-
.../master/web/handler/WebMasterInfoHandler.java | 233 ++++-
.../apache/tubemq/server/tools/BrokerStartup.java | 24 +-
.../org/apache/tubemq/server/tools/CliUtils.java | 72 ++
.../apache/tubemq/server/tools/MasterStartup.java | 21 +-
.../org/apache/tubemq/server/tools/ToolUtils.java | 78 --
.../tubemq/server/tools/cli/CliAbstractBase.java | 76 ++
.../tubemq/server/tools/cli/CliBrokerAdmin.java | 113 +++
.../tubemq/server/tools/cli/CliConsumer.java | 395 ++++++++
.../tubemq/server/tools/cli/CliProducer.java | 373 +++++++
119 files changed, 6996 insertions(+), 1538 deletions(-)
create mode 100644 DISCLAIMER
delete mode 100644 DISCLAIMER-WIP
copy bin/{groupAdmin.sh => tubemq-broker-admin.sh} (92%)
copy bin/{groupAdmin.sh => tubemq-consumer-test.sh} (92%)
copy bin/{groupAdmin.sh => tubemq-producer-test.sh} (92%)
create mode 100644 tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java
create mode 100644 tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/RegexDef.java
copy tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/{ConcurrentHashSet.java => SettingValidUtils.java} (56%)
copy tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZooKeeperConnectionException.java => tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java (51%)
copy tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/{ConcurrentHashSet.java => Tuple3.java} (50%)
create mode 100644 tubemq-manager/conf/logback.xml
create mode 100644 tubemq-manager/src/main/assembly/assembly.xml
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/HttpUtils.java
copy tubemq-server/src/main/java/org/apache/tubemq/server/common/{paramcheck/ParamCheckResult.java => utils/ProcessResult.java} (51%)
copy tubemq-core/src/main/java/org/apache/tubemq/corebase/balance/EventStatus.java => tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java (56%)
rename tubemq-server/src/main/java/org/apache/tubemq/server/{master/web/handler/WebApiMapper.java => common/webbase/WebMethodMapper.java} (57%)
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/tools/CliUtils.java
delete mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliBrokerAdmin.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
create mode 100644 tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
[incubator-tubemq] 27/49: [TUBEMQ-495]Code implementation
adjustment based on SpotBugs check
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit e8bd45a148ce617056b5559635351f3a3da0d00a
Author: gosonzhang <go...@tencent.com>
AuthorDate: Thu Jan 7 11:02:29 2021 +0800
[TUBEMQ-495]Code implementation adjustment based on SpotBugs check
---
.../tubemq/client/config/TubeClientConfig.java | 2 +-
.../client/consumer/MessageFetchManager.java | 5 +++-
.../tubemq/client/consumer/RmtDataCache.java | 4 +--
.../apache/tubemq/corebase/utils/AddressUtils.java | 12 ++++----
.../org/apache/tubemq/corebase/utils/Tuple2.java | 12 ++++++--
.../org/apache/tubemq/corebase/utils/Tuple3.java | 18 ++++++++++--
.../tubemq/corerpc/AbstractServiceInvoker.java | 2 +-
.../apache/tubemq/corerpc/RpcServiceFactory.java | 4 +--
.../tubemq/example/MAMessageProducerExample.java | 6 ++--
.../tubemq/example/MessageProducerExample.java | 6 ++--
.../server/broker/offset/DefaultOffsetManager.java | 24 ++++++++--------
.../server/broker/stats/GroupCountService.java | 2 +-
.../server/broker/utils/GroupOffsetInfo.java | 4 +--
.../server/broker/web/BrokerAdminServlet.java | 15 ++++------
.../tubemq/server/common/webbase/WebFieldType.java | 4 +--
.../org/apache/tubemq/server/master/TMaster.java | 32 +++++++++++-----------
.../master/bdbstore/DefaultBdbStoreService.java | 4 +--
.../nodemanage/nodebroker/BrokerInfoHolder.java | 4 +--
.../tubemq/server/tools/cli/CliProducer.java | 6 ++--
19 files changed, 91 insertions(+), 75 deletions(-)
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java b/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java
index b1bd3b1..437b171 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java
@@ -101,7 +101,7 @@ public class TubeClientConfig {
throw new IllegalArgumentException("Illegal parameter: masterAddrInfo is null!");
}
this.masterInfo = masterInfo.clone();
- String iPv4LocalAddress = AddressUtils.getIPV4LocalAddress();
+ AddressUtils.getIPV4LocalAddress();
}
@Deprecated
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/MessageFetchManager.java b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/MessageFetchManager.java
index 6afb8a9..80279ff 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/MessageFetchManager.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/MessageFetchManager.java
@@ -216,7 +216,10 @@ public class MessageFetchManager {
sBuilder.delete(0, sBuilder.length());
}
fetchWorkerStatusMap.put(curThreadId, 2);
- MessageFetchManager.this.pushConsumer.processRequest(partSelectResult, sBuilder);
+ if (partSelectResult != null) {
+ MessageFetchManager.this.pushConsumer.processRequest(
+ partSelectResult, sBuilder);
+ }
}
fetchWorkerStatusMap.remove(curThreadId);
}
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/RmtDataCache.java b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/RmtDataCache.java
index 745f127..a222913 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/RmtDataCache.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/RmtDataCache.java
@@ -423,8 +423,8 @@ public class RmtDataCache implements Closeable {
if (frozenTime == null) {
if (waitDlt > 10) {
TimeoutTask timeoutTask = new TimeoutTask(partitionKey);
- timeouts.put(partitionKey,
- timer.newTimeout(timeoutTask, waitDlt, TimeUnit.MILLISECONDS));
+ timeouts.put(partitionKey, timer.newTimeout(
+ timeoutTask, waitDlt, TimeUnit.MILLISECONDS));
} else {
releaseIdlePartition(partitionKey);
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
index 5a76af2..bc14e5f 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
@@ -58,12 +58,12 @@ public class AddressUtils {
try {
Tuple2<Boolean, String> result =
getValidIPV4Address(allInterface.nextElement(), currLocalHost);
- if (result.f0) {
+ if (result.getF0()) {
localIPAddress = currLocalHost;
return true;
}
if (TStringUtils.isEmpty(fstV4IP)) {
- fstV4IP = result.f1;
+ fstV4IP = result.getF1();
}
} catch (Throwable e) {
//
@@ -153,8 +153,8 @@ public class AddressUtils {
try {
Tuple2<Boolean, String> result =
getValidIPV4Address(enumeration.nextElement(), null);
- if (result.f0) {
- tmpAdress = result.f1;
+ if (result.getF0()) {
+ tmpAdress = result.getF1();
break;
}
} catch (Throwable e) {
@@ -196,8 +196,8 @@ public class AddressUtils {
try {
Tuple2<Boolean, String> result =
getValidIPV4Address(oneInterface, null);
- if (result.f0) {
- localIPAddress = result.f1;
+ if (result.getF0()) {
+ localIPAddress = result.getF1();
return localIPAddress;
}
} catch (Throwable e) {
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java
index f5626f8..048452f 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java
@@ -20,9 +20,9 @@ package org.apache.tubemq.corebase.utils;
public class Tuple2<T0, T1> {
/** Field 0 of the tuple. */
- public T0 f0 = null;
+ private T0 f0 = null;
/** Field 1 of the tuple. */
- public T1 f1 = null;
+ private T1 f1 = null;
/**
* Creates a new tuple where all fields are null.
@@ -50,4 +50,12 @@ public class Tuple2<T0, T1> {
this.f0 = value0;
this.f1 = value1;
}
+
+ public T0 getF0() {
+ return f0;
+ }
+
+ public T1 getF1() {
+ return f1;
+ }
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java
index a2d98c3..579b425 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java
@@ -20,11 +20,11 @@ package org.apache.tubemq.corebase.utils;
public class Tuple3<T0, T1, T2> {
/** Field 0 of the tuple. */
- public T0 f0 = null;
+ private T0 f0 = null;
/** Field 1 of the tuple. */
- public T1 f1 = null;
+ private T1 f1 = null;
/** Field 2 of the tuple. */
- public T2 f2 = null;
+ private T2 f2 = null;
/**
* Creates a new tuple where all fields are null.
@@ -45,4 +45,16 @@ public class Tuple3<T0, T1, T2> {
this.f1 = value1;
this.f2 = value2;
}
+
+ public T0 getF0() {
+ return f0;
+ }
+
+ public T1 getF1() {
+ return f1;
+ }
+
+ public T2 getF2() {
+ return f2;
+ }
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corerpc/AbstractServiceInvoker.java b/tubemq-core/src/main/java/org/apache/tubemq/corerpc/AbstractServiceInvoker.java
index 8103387..1154f12 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corerpc/AbstractServiceInvoker.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corerpc/AbstractServiceInvoker.java
@@ -64,7 +64,7 @@ public abstract class AbstractServiceInvoker implements InvocationHandler {
// client.close();
}
- private class RpcResponseCallback implements Callback {
+ private static class RpcResponseCallback implements Callback {
private Callback chainedCallback;
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcServiceFactory.java b/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcServiceFactory.java
index 8652c2e..cd16253 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcServiceFactory.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcServiceFactory.java
@@ -470,7 +470,7 @@ public class RpcServiceFactory {
.append(masterInfo.getMasterClusterStr()).toString();
}
- private class ServiceHolder<T> implements Shutdownable {
+ private static class ServiceHolder<T> implements Shutdownable {
private T service;
private AbstractServiceInvoker invoker;
@@ -489,7 +489,7 @@ public class RpcServiceFactory {
}
}
- private class ConnectionNode {
+ private static class ConnectionNode {
private Class clazzType;
private NodeAddrInfo addressInfo;
private RpcConfig config;
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
index ea546c9..814dbad 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
@@ -191,12 +191,12 @@ public class MAMessageProducerExample {
long millis = System.currentTimeMillis();
roundIndex = (int) (sentCount++ % targetCnt);
Tuple2<String, String> target = topicSendRounds.get(roundIndex);
- Message message = new Message(target.f0, sendData);
+ Message message = new Message(target.getF0(), sendData);
message.setAttrKeyVal("index", String.valueOf(sentCount));
message.setAttrKeyVal("dataTime", String.valueOf(millis));
- if (target.f1 != null) {
+ if (target.getF1() != null) {
filterMsgCount.incrementAndGet();
- message.putSystemHeader(target.f1, sdf.format(new Date(millis)));
+ message.putSystemHeader(target.getF1(), sdf.format(new Date(millis)));
}
try {
// next line sends message synchronously, which is not recommended
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
index 7c29a63..2b806b5 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
@@ -104,12 +104,12 @@ public final class MessageProducerExample {
while (msgCount < 0 || sentCount < msgCount) {
roundIndex = (int) (sentCount++ % targetCnt);
Tuple2<String, String> target = topicSendRounds.get(roundIndex);
- Message message = new Message(target.f0, body.getBytes());
+ Message message = new Message(target.getF0(), body.getBytes());
long currTimeMillis = System.currentTimeMillis();
message.setAttrKeyVal("index", String.valueOf(sentCount));
message.setAttrKeyVal("dataTime", String.valueOf(currTimeMillis));
- if (target.f1 != null) {
- message.putSystemHeader(target.f1, sdf.format(new Date(currTimeMillis)));
+ if (target.getF1() != null) {
+ message.putSystemHeader(target.getF1(), sdf.format(new Date(currTimeMillis)));
}
try {
// 1.1 next line sends message synchronously, which is not recommended
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
index f052375..0e91dc2 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
@@ -439,10 +439,12 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
getOffsetCacheKey(entry.getKey(), partitionId);
OffsetStorageInfo offsetInfo = topicPartOffsetMap.get(offsetCacheKey);
Long tmpOffset = tmpPartOffsetMap.get(offsetCacheKey);
+ if (tmpOffset == null) {
+ tmpOffset = 0L;
+ }
if (offsetInfo != null) {
offsetMap.put(partitionId,
- new Tuple2<>(offsetInfo.getOffset(),
- (tmpOffset == null ? 0 : tmpOffset)));
+ new Tuple2<>(offsetInfo.getOffset(), tmpOffset));
}
}
if (!offsetMap.isEmpty()) {
@@ -473,21 +475,21 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
for (String group : groups) {
for (Tuple3<String, Integer, Long> tuple3 : topicPartOffsets) {
if (tuple3 == null
- || tuple3.f0 == null
- || tuple3.f1 == null
- || tuple3.f2 == null) {
+ || tuple3.getF0() == null
+ || tuple3.getF1() == null
+ || tuple3.getF2() == null) {
continue;
}
// set offset value
- offsetCacheKey = getOffsetCacheKey(tuple3.f0, tuple3.f1);
+ offsetCacheKey = getOffsetCacheKey(tuple3.getF0(), tuple3.getF1());
getAndResetTmpOffset(group, offsetCacheKey);
OffsetStorageInfo regInfo = loadOrCreateOffset(group,
- tuple3.f0, tuple3.f1, offsetCacheKey, 0);
- oldOffset = regInfo.getAndSetOffset(tuple3.f2);
+ tuple3.getF0(), tuple3.getF1(), offsetCacheKey, 0);
+ oldOffset = regInfo.getAndSetOffset(tuple3.getF2());
changed = true;
logger.info(strBuidler
.append("[Offset Manager] Update offset by modifier=")
- .append(modifier).append(",reset offset=").append(tuple3.f2)
+ .append(modifier).append(",reset offset=").append(tuple3.getF2())
.append(",old offset=").append(oldOffset)
.append(",updated offset=").append(regInfo.getOffset())
.append(",group=").append(group)
@@ -710,9 +712,5 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
.append("-").append(partitionId).toString();
}
- private String getOffsetCacheKey(String topic, String partitionId) {
- return new StringBuilder(256).append(topic)
- .append("-").append(partitionId).toString();
- }
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/stats/GroupCountService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/stats/GroupCountService.java
index b0de32e..7a7c056 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/stats/GroupCountService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/stats/GroupCountService.java
@@ -139,7 +139,7 @@ public class GroupCountService extends AbstractDaemonService implements CountSer
countSet.refCnt.decrementAndGet();
}
- private class CountSet {
+ private static class CountSet {
public AtomicLong refCnt = new AtomicLong(0);
public ConcurrentHashMap<String, CountItem> counterItem =
new ConcurrentHashMap<>();
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
index a0c7215..fc0ebfa 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
@@ -48,8 +48,8 @@ public class GroupOffsetInfo {
public void setConsumeOffsetInfo(Tuple2<Long, Long> offsetInfo) {
if (offsetInfo != null) {
- this.curOffset = offsetInfo.f0;
- this.flightOffset = offsetInfo.f1;
+ this.curOffset = offsetInfo.getF0();
+ this.flightOffset = offsetInfo.getF1();
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index 30d7247..5b11be1 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -459,7 +459,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
return;
}
Set<String> filterCondStrSet = (Set<String>) result.retData1;
- sBuilder = broker.getBrokerServiceServer()
+ broker.getBrokerServiceServer()
.getMessageSnapshot(topicName, partitionId, msgCount, filterCondStrSet, sBuilder);
}
@@ -811,8 +811,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// transfer offset format
resetOffsets = buildOffsetResetInfo(topicSet);
}
- boolean changed = broker.getOffsetManager().modifyGroupOffset(
- groupNameSet, resetOffsets, modifier);
+ broker.getOffsetManager().modifyGroupOffset(groupNameSet, resetOffsets, modifier);
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
}
@@ -875,10 +874,8 @@ public class BrokerAdminServlet extends AbstractWebHandler {
Map<String, Map<Integer, Tuple2<Long, Long>>> srcGroupOffsets =
broker.getOffsetManager().queryGroupOffset(srcGroupName, topicPartMap);
// transfer offset format
- List<Tuple3<String, Integer, Long>> resetOffsets =
- buildOffsetResetInfo(srcGroupOffsets);
- boolean changed = broker.getOffsetManager().modifyGroupOffset(
- tgtGroupNameSet, resetOffsets, modifier);
+ List<Tuple3<String, Integer, Long>> resetOffsets = buildOffsetResetInfo(srcGroupOffsets);
+ broker.getOffsetManager().modifyGroupOffset(tgtGroupNameSet, resetOffsets, modifier);
// builder return result
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
}
@@ -965,8 +962,8 @@ public class BrokerAdminServlet extends AbstractWebHandler {
long firstOffset = store.getIndexMinOffset();
long lastOffset = store.getIndexMaxOffset();
// adjust reset offset value
- adjOffset = offsetTuple.f0 < firstOffset
- ? firstOffset : Math.min(offsetTuple.f0, lastOffset);
+ adjOffset = offsetTuple.getF0() < firstOffset
+ ? firstOffset : Math.min(offsetTuple.getF0(), lastOffset);
result.add(new Tuple3<>(entry.getKey(), entry1.getKey(), adjOffset));
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
index 2f32cb1..a3e037f 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
@@ -32,8 +32,8 @@ public enum WebFieldType {
JSONTYPE(8, "Json");
- public int value;
- public String desc;
+ private int value;
+ private String desc;
WebFieldType(int value, String desc) {
this.value = value;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
index cc6f681..4299bc7 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
@@ -1956,12 +1956,12 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
Tuple2<String, ConsumerInfo> tupleInfo =
consumerHolder.getConsumeTupleInfo(consumerId);
if (tupleInfo == null
- || tupleInfo.f0 == null
- || tupleInfo.f1 == null) {
+ || tupleInfo.getF0() == null
+ || tupleInfo.getF1() == null) {
continue;
}
List<String> blackTopicList =
- this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
+ this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.getF0());
Map<String, List<Partition>> topicSubPartMap = entry.getValue();
List<SubscribeInfo> deletedSubInfoList = new ArrayList<>();
List<SubscribeInfo> addedSubInfoList = new ArrayList<>();
@@ -1979,7 +1979,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
currentPartMap = new HashMap<>();
}
}
- if (tupleInfo.f1.isOverTLS()) {
+ if (tupleInfo.getF1().isOverTLS()) {
for (Partition currentPart : currentPartMap.values()) {
if (!blackTopicList.contains(currentPart.getTopic())) {
boolean found = false;
@@ -1995,8 +1995,8 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
}
}
deletedSubInfoList
- .add(new SubscribeInfo(consumerId, tupleInfo.f0,
- tupleInfo.f1.isOverTLS(), currentPart));
+ .add(new SubscribeInfo(consumerId, tupleInfo.getF0(),
+ tupleInfo.getF1().isOverTLS(), currentPart));
}
for (Partition finalPart : finalPartList) {
if (!blackTopicList.contains(finalPart.getTopic())) {
@@ -2012,7 +2012,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
continue;
}
addedSubInfoList.add(new SubscribeInfo(consumerId,
- tupleInfo.f0, true, finalPart));
+ tupleInfo.getF0(), true, finalPart));
}
}
} else {
@@ -2020,14 +2020,14 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if ((blackTopicList.contains(currentPart.getTopic()))
|| (!finalPartList.contains(currentPart))) {
deletedSubInfoList.add(new SubscribeInfo(consumerId,
- tupleInfo.f0, false, currentPart));
+ tupleInfo.getF0(), false, currentPart));
}
}
for (Partition finalPart : finalPartList) {
if ((currentPartMap.get(finalPart.getPartitionKey()) == null)
&& (!blackTopicList.contains(finalPart.getTopic()))) {
addedSubInfoList.add(new SubscribeInfo(consumerId,
- tupleInfo.f0, false, finalPart));
+ tupleInfo.getF0(), false, finalPart));
}
}
}
@@ -2090,13 +2090,13 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
Tuple2<String, ConsumerInfo> tupleInfo =
consumerHolder.getConsumeTupleInfo(consumerId);
if (tupleInfo == null
- || tupleInfo.f0 == null
- || tupleInfo.f1 == null) {
+ || tupleInfo.getF0() == null
+ || tupleInfo.getF1() == null) {
continue;
}
// allocate partitions to consumers
List<String> blackTopicList =
- this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
+ this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.getF0());
Map<String, Map<String, Partition>> topicSubPartMap = entry.getValue();
List<SubscribeInfo> deletedSubInfoList = new ArrayList<>();
List<SubscribeInfo> addedSubInfoList = new ArrayList<>();
@@ -2120,15 +2120,15 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if ((blackTopicList.contains(currentPart.getTopic()))
|| (finalPartMap.get(currentPart.getPartitionKey()) == null)) {
deletedSubInfoList
- .add(new SubscribeInfo(consumerId, tupleInfo.f0,
- tupleInfo.f1.isOverTLS(), currentPart));
+ .add(new SubscribeInfo(consumerId, tupleInfo.getF0(),
+ tupleInfo.getF1().isOverTLS(), currentPart));
}
}
for (Partition finalPart : finalPartMap.values()) {
if ((currentPartMap.get(finalPart.getPartitionKey()) == null)
&& (!blackTopicList.contains(finalPart.getTopic()))) {
- addedSubInfoList.add(new SubscribeInfo(consumerId, tupleInfo.f0,
- tupleInfo.f1.isOverTLS(), finalPart));
+ addedSubInfoList.add(new SubscribeInfo(consumerId, tupleInfo.getF0(),
+ tupleInfo.getF1().isOverTLS(), finalPart));
}
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java
index 7622c57..b201cde 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java
@@ -930,9 +930,7 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
if (!isPrimaryNodeActive()) {
if ((replicas4Transfer != null) && (!replicas4Transfer.isEmpty())) {
logger.info("start transferMaster to replicas: " + replicas4Transfer);
- if ((replicas4Transfer != null) && (!replicas4Transfer.isEmpty())) {
- repEnv.transferMaster(replicas4Transfer, 5, TimeUnit.MINUTES);
- }
+ repEnv.transferMaster(replicas4Transfer, 5, TimeUnit.MINUTES);
logger.info("transferMaster end...");
} else {
throw new Exception("The replicate nodes is empty!");
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerInfoHolder.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerInfoHolder.java
index dec535b..c9fdaea 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerInfoHolder.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerInfoHolder.java
@@ -323,7 +323,7 @@ public class BrokerInfoHolder {
}
}
- public class BrokerAbnInfo {
+ public static class BrokerAbnInfo {
private int brokerId;
private int abnStatus; // 0 normal , -100 read abnormal, -1 write abnormal, -101 r & w abnormal
private long firstRepTime;
@@ -364,7 +364,7 @@ public class BrokerInfoHolder {
}
}
- public class BrokerFbdInfo {
+ public static class BrokerFbdInfo {
private int brokerId;
private int befStatus;
private int newStatus;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
index f544829..a6e7e38 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
@@ -268,10 +268,10 @@ public class CliProducer extends CliAbstractBase {
try {
long millis = System.currentTimeMillis();
Tuple2<String, String> target = topicSendRounds.get(roundIndex);
- Message message = new Message(target.f0, sentData);
- if (target.f1 != null) {
+ Message message = new Message(target.getF0(), sentData);
+ if (target.getF1() != null) {
// if include filter, add filter item
- message.putSystemHeader(target.f1, sdf.format(new Date(millis)));
+ message.putSystemHeader(target.getF1(), sdf.format(new Date(millis)));
}
// use sync or async process
if (syncProduction) {
[incubator-tubemq] 35/49: [TUBEMQ-509] Adjust the packet length
check when data is loaded
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit d25d00c43561b4a5448dca0affa57ec0a9071f9a
Author: gosonzhang <go...@tencent.com>
AuthorDate: Tue Jan 12 18:50:27 2021 +0800
[TUBEMQ-509] Adjust the packet length check when data is loaded
---
.../org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
index 134e2ef..1b370d5 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.protobuf.generated.ClientMaster;
import org.apache.tubemq.corebase.utils.MixedUtils;
-import org.apache.tubemq.server.broker.utils.DataStoreUtils;
+
public class ClusterConfigHolder {
[incubator-tubemq] 05/49: [TUBEMQ-442]Modifying the jvm parameters
when the broker starts does not take effect (#340)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit a835a50130ccbf0a7746b66544617d7f80091e0b
Author: gosonzhang <46...@qq.com>
AuthorDate: Fri Dec 4 17:37:16 2020 +0800
[TUBEMQ-442]Modifying the jvm parameters when the broker starts does not take effect (#340)
Co-authored-by: gosonzhang <go...@tencent.com>
---
bin/env.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/env.sh b/bin/env.sh
index 46d168c..4364993 100644
--- a/bin/env.sh
+++ b/bin/env.sh
@@ -35,7 +35,7 @@ if [ -z "$MASTER_JVM_SIZE" ]; then
fi
MASTER_JVM_ARGS="$MASTER_JVM_SIZE -server -Dtubemq.home=$tubemq_home -cp $CLASSPATH "
#Broker jvm args
-if [ -z "$MASTER_JVM_SIZE" ]; then
+if [ -z "$BROKER_JVM_SIZE" ]; then
BROKER_JVM_SIZE="-Xmx16g -Xms8g"
fi
BROKER_JVM_ARGS="$BROKER_JVM_SIZE -server -Dtubemq.home=$tubemq_home -cp $CLASSPATH "
[incubator-tubemq] 29/49: [TUBEMQ-504]Adjust the WebMethodMapper
class interfaces (#388)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 8762bd13afc000b8177465eafe23e6fb7082b504
Author: gosonzhang <46...@qq.com>
AuthorDate: Sat Jan 9 21:49:54 2021 +0800
[TUBEMQ-504]Adjust the WebMethodMapper class interfaces (#388)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../server/broker/web/AbstractWebHandler.java | 13 +++---
.../server/broker/web/BrokerAdminServlet.java | 36 ++++++++--------
.../tubemq/server/common/utils/ProcessResult.java | 7 ++++
.../server/common/utils/WebParameterUtils.java | 14 ++++---
.../server/common/webbase/WebMethodMapper.java | 48 +++++++++-------------
.../server/master/web/action/screen/Webapi.java | 2 +-
.../master/web/handler/AbstractWebHandler.java | 14 ++++---
7 files changed, 71 insertions(+), 63 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
index 508849c..17a378d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
@@ -21,7 +21,8 @@ import static org.apache.tubemq.server.common.webbase.WebMethodMapper.getRegiste
import static org.apache.tubemq.server.common.webbase.WebMethodMapper.getWebApiRegInfo;
import static org.apache.tubemq.server.common.webbase.WebMethodMapper.registerWebMethod;
import java.io.IOException;
-import java.util.List;
+import java.util.Set;
+
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -44,7 +45,7 @@ public abstract class AbstractWebHandler extends HttpServlet {
doPost(req, resp);
}
- public List<String> getSupportedMethod() {
+ public Set<String> getSupportedMethod() {
return getRegisteredWebMethod();
}
@@ -59,7 +60,7 @@ public abstract class AbstractWebHandler extends HttpServlet {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Please take with method parameter! \"}");
} else {
- WebApiRegInfo webApiRegInfo = getWebApiRegInfo(true, method);
+ WebApiRegInfo webApiRegInfo = getWebApiRegInfo(method);
if (webApiRegInfo == null) {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Unsupported method ").append(method).append("\"}");
@@ -82,8 +83,10 @@ public abstract class AbstractWebHandler extends HttpServlet {
public abstract void registerWebApiMethod();
protected void innRegisterWebMethod(String webMethodName,
- String clsMethodName) {
- registerWebMethod(true, webMethodName, clsMethodName, this);
+ String clsMethodName,
+ boolean needAuthToken) {
+ registerWebMethod(webMethodName, clsMethodName,
+ false, needAuthToken, this);
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index 5b11be1..d35ba91 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -58,59 +58,59 @@ public class BrokerAdminServlet extends AbstractWebHandler {
public void registerWebApiMethod() {
// query consumer group's offset
innRegisterWebMethod("admin_query_group_offset",
- "adminQueryCurrentGroupOffSet");
+ "adminQueryCurrentGroupOffSet", false);
// query snapshot message
innRegisterWebMethod("admin_snapshot_message",
- "adminQuerySnapshotMessageSet");
+ "adminQuerySnapshotMessageSet", false);
// query broker's all consumer info
innRegisterWebMethod("admin_query_broker_all_consumer_info",
- "adminQueryBrokerAllConsumerInfo");
+ "adminQueryBrokerAllConsumerInfo", false);
// get memory store status info
innRegisterWebMethod("admin_query_broker_memstore_info",
- "adminGetMemStoreStatisInfo");
+ "adminGetMemStoreStatisInfo", false);
// query broker's all message store info
innRegisterWebMethod("admin_query_broker_all_store_info",
- "adminQueryBrokerAllMessageStoreInfo");
+ "adminQueryBrokerAllMessageStoreInfo", false);
// query consumer register info
innRegisterWebMethod("admin_query_consumer_regmap",
- "adminQueryConsumerRegisterInfo");
+ "adminQueryConsumerRegisterInfo", false);
// manual set offset
innRegisterWebMethod("admin_manual_set_current_offset",
- "adminManualSetCurrentOffSet");
+ "adminManualSetCurrentOffSet", false);
// get all registered methods
innRegisterWebMethod("admin_get_methods",
- "adminQueryAllMethods");
+ "adminQueryAllMethods", false);
// query topic's publish info
innRegisterWebMethod("admin_query_pubinfo",
- "adminQueryPubInfo");
+ "adminQueryPubInfo", false);
// Query all consumer groups booked on the Broker.
innRegisterWebMethod("admin_query_group",
- "adminQueryBookedGroup");
+ "adminQueryBookedGroup", false);
// query consumer group's offset
innRegisterWebMethod("admin_query_offset",
- "adminQueryGroupOffSet");
+ "adminQueryGroupOffSet", false);
// clone consumer group's offset from source to target
innRegisterWebMethod("admin_clone_offset",
- "adminCloneGroupOffSet");
+ "adminCloneGroupOffSet", false);
// set or update group's offset info
innRegisterWebMethod("admin_set_offset",
- "adminSetGroupOffSet");
+ "adminSetGroupOffSet", false);
// remove group's offset info
innRegisterWebMethod("admin_rmv_offset",
- "adminRemoveGroupOffSet");
+ "adminRemoveGroupOffSet", false);
}
public void adminQueryAllMethods(HttpServletRequest req,
StringBuilder sBuilder) {
int index = 0;
- List<String> methods = getSupportedMethod();
+ Set<String> methods = getSupportedMethod();
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
- for (index = 0; index < methods.size(); index++) {
- if (index > 0) {
+ for (String method : methods) {
+ if (index++ > 0) {
sBuilder.append(",");
}
sBuilder.append("{\"id\":").append(index + 1)
- .append(",\"method\":\"").append(methods.get(index)).append("\"}");
+ .append(",\"method\":\"").append(method).append("\"}");
}
sBuilder.append("],\"totalCnt\":").append(index + 1).append("}");
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
index 8cabf67..5b13dd8 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
@@ -47,6 +47,13 @@ public class ProcessResult {
this.retData1 = null;
}
+ public void setFailResult(final String errMsg) {
+ this.success = false;
+ this.errCode = TErrCodeConstants.BAD_REQUEST;
+ this.errInfo = errMsg;
+ this.retData1 = null;
+ }
+
public void setSuccResult(Object retData) {
this.success = true;
this.errInfo = "";
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index 2318bdb..f309ab7 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -242,13 +242,17 @@ public class WebParameterUtils {
.append(errMsg).append("\"}");
}
+ public static StringBuilder buildSuccessResult(StringBuilder strBuffer) {
+ return strBuffer.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
+ }
+
/**
* Parse the parameter value from an object value to a long value
*
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
- * @param defValue a default value returned if failed to parse value from the given object
+ * @param defValue a default value returned if the field not exist
* @param result process result of parameter value
* @return process result
*/
@@ -329,7 +333,7 @@ public class WebParameterUtils {
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
- * @param defValue a default value returned if failed to parse value from the given object
+ * @param defValue a default value returned if the field not exist
* @param minValue min value required
* @param result process result of parameter value
* @return process result
@@ -377,7 +381,7 @@ public class WebParameterUtils {
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
- * @param defValue a default value returned if failed to parse value from the given object
+ * @param defValue a default value returned if the field not exist
* @param result process result
* @return valid result for the parameter value
*/
@@ -404,7 +408,7 @@ public class WebParameterUtils {
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
- * @param defValue a default value returned if failed to parse value from the given object
+ * @param defValue a default value returned if the field not exist
* @param result process result
* @return valid result for the parameter value
*/
@@ -486,7 +490,7 @@ public class WebParameterUtils {
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
- * @param defValue a default value returned if failed to parse value from the given object
+ * @param defValue a default value returned if the field not exist
* @param result process result
* @return valid result for the parameter value
*/
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
index 83e0472..147a388 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
@@ -18,10 +18,9 @@
package org.apache.tubemq.server.common.webbase;
import java.lang.reflect.Method;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,36 +29,25 @@ public class WebMethodMapper {
// log printer
private static final Logger logger =
LoggerFactory.getLogger(WebMethodMapper.class);
- // The query methods map
- public static final Map<String, WebApiRegInfo> WEB_QRY_METHOD_MAP =
- new HashMap<>();
- // The modify methods map
- public static final Map<String, WebApiRegInfo> WEB_MDY_METHOD_MAP =
+
+ public static final Map<String, WebApiRegInfo> WEB_METHOD_MAP =
new HashMap<>();
- public static WebApiRegInfo getWebApiRegInfo(boolean isQryApi,
- String webMethodName) {
- if (isQryApi) {
- return WEB_QRY_METHOD_MAP.get(webMethodName);
- }
- return WEB_MDY_METHOD_MAP.get(webMethodName);
+ public static WebApiRegInfo getWebApiRegInfo(String webMethodName) {
+ return WEB_METHOD_MAP.get(webMethodName);
}
- public static void registerWebMethod(boolean isQryApi,
- String webMethodName,
+ public static void registerWebMethod(String webMethodName,
String clsMethodName,
+ boolean onlyMasterOp,
+ boolean needAuthToken,
Object webHandler) {
Method[] methods = webHandler.getClass().getMethods();
for (Method item : methods) {
if (item.getName().equals(clsMethodName)) {
- if (isQryApi) {
- WEB_QRY_METHOD_MAP.put(webMethodName,
- new WebApiRegInfo(item, webHandler));
- } else {
- WEB_MDY_METHOD_MAP.put(webMethodName,
- new WebApiRegInfo(item, webHandler));
- }
+ WEB_METHOD_MAP.put(webMethodName,
+ new WebApiRegInfo(item, webHandler, onlyMasterOp, needAuthToken));
return;
}
}
@@ -69,11 +57,8 @@ public class WebMethodMapper {
.append(webHandler.getClass().getName()).toString());
}
- public static List<String> getRegisteredWebMethod() {
- List<String> methods = new ArrayList<>();
- methods.addAll(WEB_QRY_METHOD_MAP.keySet());
- methods.addAll(WEB_MDY_METHOD_MAP.keySet());
- return methods;
+ public static Set<String> getRegisteredWebMethod() {
+ return WEB_METHOD_MAP.keySet();
}
@@ -81,11 +66,18 @@ public class WebMethodMapper {
public static class WebApiRegInfo {
public Method method;
public Object webHandler;
+ public boolean onlyMasterOp = false;
+ public boolean needAuthToken = false;
+
public WebApiRegInfo(Method method,
- Object webHandler) {
+ Object webHandler,
+ boolean onlyMasterOp,
+ boolean needAuthToken) {
this.method = method;
this.webHandler = webHandler;
+ this.onlyMasterOp = onlyMasterOp;
+ this.needAuthToken = needAuthToken;
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
index 5c5bb3f..126f1e8 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
@@ -102,7 +102,7 @@ public class Webapi implements Action {
"DesignatedPrimary happened...please check if the other member is down");
}
}
- WebMethodMapper.WebApiRegInfo webApiRegInfo = getWebApiRegInfo(isQuery, method);
+ WebMethodMapper.WebApiRegInfo webApiRegInfo = getWebApiRegInfo(method);
if (webApiRegInfo == null) {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"Unsupported method: ")
.append(method).append("\"}");
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java
index 1b1bfdc..63de11c 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java
@@ -37,18 +37,20 @@ public abstract class AbstractWebHandler {
protected void registerQueryWebMethod(String webMethodName,
String clsMethodName) {
- innRegisterWebMethod(true, webMethodName, clsMethodName);
+ innRegisterWebMethod(webMethodName, clsMethodName, false, false);
}
protected void registerModifyWebMethod(String webMethodName,
String clsMethodName) {
- innRegisterWebMethod(false, webMethodName, clsMethodName);
+ innRegisterWebMethod(webMethodName, clsMethodName, true, true);
}
- private void innRegisterWebMethod(boolean isQryApi,
- String webMethodName,
- String clsMethodName) {
- registerWebMethod(isQryApi, webMethodName, clsMethodName, this);
+ private void innRegisterWebMethod(String webMethodName,
+ String clsMethodName,
+ boolean onlyMasterOp,
+ boolean needAuthToken) {
+ registerWebMethod(webMethodName, clsMethodName,
+ onlyMasterOp, needAuthToken, this);
}
}
[incubator-tubemq] 25/49: [TUBEMQ-486]Add the delete API of
consumer group offset
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit e3d818c2759bef5f6d84ebeb13a212b4e7493fd9
Author: gosonzhang <go...@tencent.com>
AuthorDate: Wed Jan 6 11:27:15 2021 +0800
[TUBEMQ-486]Add the delete API of consumer group offset
---
.../server/broker/offset/DefaultOffsetManager.java | 104 ++++-
.../tubemq/server/broker/offset/OffsetService.java | 4 +
.../server/broker/web/BrokerAdminServlet.java | 430 +++++++++++++--------
.../tubemq/server/common/fielddef/WebFieldDef.java | 6 +-
.../tubemq/server/common/fileconfig/ZKConfig.java | 1 -
.../server/common/offsetstorage/OffsetStorage.java | 7 +-
.../common/offsetstorage/ZkOffsetStorage.java | 175 ++++++---
.../common/offsetstorage/zookeeper/ZKUtil.java | 14 +
.../tubemq/server/common/utils/ProcessResult.java | 3 +
.../server/common/utils/WebParameterUtils.java | 278 ++++++-------
.../web/handler/WebBrokerTopicConfHandler.java | 21 +-
11 files changed, 661 insertions(+), 382 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
index 84dabb2..f052375 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
@@ -17,6 +17,7 @@
package org.apache.tubemq.server.broker.offset;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -339,7 +340,7 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
Set<String> groupSet = new HashSet<>();
groupSet.addAll(cfmOffsetMap.keySet());
Map<String, Set<String>> localGroups =
- zkOffsetStorage.getZkLocalGroupTopicInfos();
+ zkOffsetStorage.queryZkAllGroupTopicInfos();
groupSet.addAll(localGroups.keySet());
return groupSet;
}
@@ -364,7 +365,7 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
public Set<String> getUnusedGroupInfo() {
Set<String> unUsedGroups = new HashSet<>();
Map<String, Set<String>> localGroups =
- zkOffsetStorage.getZkLocalGroupTopicInfos();
+ zkOffsetStorage.queryZkAllGroupTopicInfos();
for (String groupName : localGroups.keySet()) {
if (!cfmOffsetMap.containsKey(groupName)) {
unUsedGroups.add(groupName);
@@ -383,9 +384,11 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
Set<String> result = new HashSet<>();
Map<String, OffsetStorageInfo> topicPartOffsetMap = cfmOffsetMap.get(group);
if (topicPartOffsetMap == null) {
- Map<String, Set<String>> localGroups =
- zkOffsetStorage.getZkLocalGroupTopicInfos();
- result = localGroups.get(group);
+ List<String> groupLst = new ArrayList<>(1);
+ groupLst.add(group);
+ Map<String, Set<String>> groupTopicInfo =
+ zkOffsetStorage.queryZKGroupTopicInfo(groupLst);
+ result = groupTopicInfo.get(group);
} else {
for (OffsetStorageInfo storageInfo : topicPartOffsetMap.values()) {
result.add(storageInfo.getTopic());
@@ -496,6 +499,53 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
}
/***
+ * Delete offset.
+ *
+ * @param onlyMemory
+ * @param groupTopicPartMap
+ * @param modifier
+ */
+ @Override
+ public void deleteGroupOffset(boolean onlyMemory,
+ Map<String, Map<String, Set<Integer>>> groupTopicPartMap,
+ String modifier) {
+ String printBase;
+ StringBuilder strBuidler = new StringBuilder(512);
+ for (Map.Entry<String, Map<String, Set<Integer>>> entry
+ : groupTopicPartMap.entrySet()) {
+ if (entry.getKey() == null
+ || entry.getValue() == null
+ || entry.getValue().isEmpty()) {
+ continue;
+ }
+ rmvOffset(entry.getKey(), entry.getValue());
+ }
+ if (onlyMemory) {
+ printBase = strBuidler
+ .append("[Offset Manager] delete offset from memory by modifier=")
+ .append(modifier).toString();
+ } else {
+ zkOffsetStorage.deleteGroupOffsetInfo(groupTopicPartMap);
+ printBase = strBuidler
+ .append("[Offset Manager] delete offset from memory and zk by modifier=")
+ .append(modifier).toString();
+ }
+ strBuidler.delete(0, strBuidler.length());
+ // print log
+ for (Map.Entry<String, Map<String, Set<Integer>>> entry
+ : groupTopicPartMap.entrySet()) {
+ if (entry.getKey() == null
+ || entry.getValue() == null
+ || entry.getValue().isEmpty()) {
+ continue;
+ }
+ logger.info(strBuidler.append(printBase).append(",group=").append(entry.getKey())
+ .append(",topic-partId-map=").append(entry.getValue()).toString());
+ strBuidler.delete(0, strBuidler.length());
+ }
+ }
+
+ /***
* Set temp offset.
*
* @param group
@@ -611,6 +661,50 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
return regInfo;
}
+ private void rmvOffset(String group, Map<String, Set<Integer>> topicPartMap) {
+ if (group == null
+ || topicPartMap == null
+ || topicPartMap.isEmpty()) {
+ return;
+ }
+ // remove confirm offset
+ ConcurrentHashMap<String, OffsetStorageInfo> regInfoMap = cfmOffsetMap.get(group);
+ if (regInfoMap != null) {
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ if (entry.getKey() == null
+ || entry.getValue() == null
+ || entry.getValue().isEmpty()) {
+ continue;
+ }
+ for (Integer partitionId : entry.getValue()) {
+ String offsetCacheKey = getOffsetCacheKey(entry.getKey(), partitionId);
+ regInfoMap.remove(offsetCacheKey);
+ }
+ }
+ if (regInfoMap.isEmpty()) {
+ cfmOffsetMap.remove(group);
+ }
+ }
+ // remove tmp offset
+ ConcurrentHashMap<String, Long> tmpRegInfoMap = tmpOffsetMap.get(group);
+ if (tmpRegInfoMap != null) {
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ if (entry.getKey() == null
+ || entry.getValue() == null
+ || entry.getValue().isEmpty()) {
+ continue;
+ }
+ for (Integer partitionId : entry.getValue()) {
+ String offsetCacheKey = getOffsetCacheKey(entry.getKey(), partitionId);
+ tmpRegInfoMap.remove(offsetCacheKey);
+ }
+ }
+ if (tmpRegInfoMap.isEmpty()) {
+ tmpOffsetMap.remove(group);
+ }
+ }
+ }
+
private String getOffsetCacheKey(String topic, int partitionId) {
return new StringBuilder(256).append(topic)
.append("-").append(partitionId).toString();
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
index 9dcd29a..4a19798 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
@@ -72,4 +72,8 @@ public interface OffsetService {
boolean modifyGroupOffset(Set<String> groups,
List<Tuple3<String, Integer, Long>> topicPartOffsets,
String modifier);
+
+ void deleteGroupOffset(boolean onlyMemory,
+ Map<String, Map<String, Set<Integer>>> groupTopicPartMap,
+ String modifier);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index c76a6b7..1f2fb5d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -95,10 +95,13 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// set or update group's offset info
innRegisterWebMethod("admin_set_offset",
"adminSetGroupOffSet");
+ // remove group's offset info
+ innRegisterWebMethod("admin_rmv_offset",
+ "adminRemoveGroupOffSet");
}
public void adminQueryAllMethods(HttpServletRequest req,
- StringBuilder sBuilder) throws Exception {
+ StringBuilder sBuilder) {
int index = 0;
List<String> methods = getSupportedMethod();
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
@@ -120,11 +123,11 @@ public class BrokerAdminServlet extends AbstractWebHandler {
* @throws Exception
*/
public void adminQueryBrokerAllConsumerInfo(HttpServletRequest req,
- StringBuilder sBuilder) throws Exception {
+ StringBuilder sBuilder) {
int index = 0;
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSGROUPNAME, false, null);
- if (!result.success) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -209,10 +212,10 @@ public class BrokerAdminServlet extends AbstractWebHandler {
* @throws Exception
*/
public void adminQueryBrokerAllMessageStoreInfo(HttpServletRequest req,
- StringBuilder sBuilder) throws Exception {
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, false, null);
- if (!result.success) {
+ StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -276,17 +279,16 @@ public class BrokerAdminServlet extends AbstractWebHandler {
* @throws Exception
*/
public void adminGetMemStoreStatisInfo(HttpServletRequest req,
- StringBuilder sBuilder) throws Exception {
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, false, null);
- if (!result.success) {
+ StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
Set<String> topicNameSet = (Set<String>) result.retData1;
- result = WebParameterUtils.getBooleanParamValue(req,
- WebFieldDef.NEEDREFRESH, false, false);
- if (!result.success) {
+ if (!WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.NEEDREFRESH, false, false, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -335,38 +337,34 @@ public class BrokerAdminServlet extends AbstractWebHandler {
* @throws Exception
*/
public void adminManualSetCurrentOffSet(HttpServletRequest req,
- StringBuilder sBuilder) throws Exception {
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.TOPICNAME, true, null);
- if (!result.success) {
+ StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TOPICNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String topicName = (String) result.retData1;
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.GROUPNAME, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.GROUPNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String groupName = (String) result.retData1;
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.MODIFYUSER, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String modifyUser = (String) result.retData1;
- result = WebParameterUtils.getIntParamValue(req,
- WebFieldDef.PARTITIONID, true, -1, 0);
- if (!result.success) {
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.PARTITIONID, true, -1, 0, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
int partitionId = (Integer) result.retData1;
- result = WebParameterUtils.getLongParamValue(req,
- WebFieldDef.MANUALOFFSET, true, -1);
- if (!result.success) {
+ if (!WebParameterUtils.getLongParamValue(req,
+ WebFieldDef.MANUALOFFSET, true, -1, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -429,23 +427,21 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminQuerySnapshotMessageSet(HttpServletRequest req,
StringBuilder sBuilder) throws Exception {
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.TOPICNAME, true, null);
- if (!result.success) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TOPICNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String topicName = (String) result.retData1;
- result = WebParameterUtils.getIntParamValue(req,
- WebFieldDef.PARTITIONID, true, -1, 0);
- if (!result.success) {
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.PARTITIONID, true, -1, 0, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
int partitionId = (Integer) result.retData1;
- result = WebParameterUtils.getIntParamValue(req,
- WebFieldDef.MSGCOUNT, false, 3, 3);
- if (!result.success) {
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.MSGCOUNT, false, 3, 3, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -457,9 +453,8 @@ public class BrokerAdminServlet extends AbstractWebHandler {
.append("\"}");
return;
}
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.FILTERCONDS, false, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.FILTERCONDS, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -477,31 +472,27 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminQueryCurrentGroupOffSet(HttpServletRequest req,
StringBuilder sBuilder) {
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.TOPICNAME, true, null);
- if (!result.success) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TOPICNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String topicName = (String) result.retData1;
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.GROUPNAME, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.GROUPNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String groupName = (String) result.retData1;
- result = WebParameterUtils.getIntParamValue(req,
- WebFieldDef.PARTITIONID, true, -1, 0);
- if (!result.success) {
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.PARTITIONID, true, -1, 0, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
int partitionId = (Integer) result.retData1;
-
- result = WebParameterUtils.getBooleanParamValue(req,
- WebFieldDef.REQUIREREALOFFSET, false, false);
- if (!result.success) {
+ if (!WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.REQUIREREALOFFSET, false, false, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -567,8 +558,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
Map<String, ConsumerNodeInfo> map =
broker.getBrokerServiceServer().getConsumerRegisterMap();
int totalCnt = 0;
- sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",")
- .append(",\"dataSet\":[");
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
for (Entry<String, ConsumerNodeInfo> entry : map.entrySet()) {
if (entry.getKey() == null || entry.getValue() == null) {
continue;
@@ -592,10 +582,10 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminQueryPubInfo(HttpServletRequest req,
StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
// get the topic set to be queried
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, false, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -637,9 +627,9 @@ public class BrokerAdminServlet extends AbstractWebHandler {
public void adminQueryBookedGroup(HttpServletRequest req,
StringBuilder sBuilder) {
// get divide info
- ProcessResult result = WebParameterUtils.getBooleanParamValue(req,
- WebFieldDef.WITHDIVIDE, false, false);
- if (!result.success) {
+ ProcessResult result = new ProcessResult();
+ if (!WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.WITHDIVIDE, false, false, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -697,16 +687,24 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminQueryGroupOffSet(HttpServletRequest req,
StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
// get group list
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSGROUPNAME, false, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
+ Set<String> inGroupNameSet = (Set<String>) result.retData1;
+ // get the topic set to be queried
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ // get target consume group name
+ Set<String> topicSet = (Set<String>) result.retData1;
// filter invalid groups
Set<String> qryGroupNameSet = new HashSet<>();
- Set<String> inGroupNameSet = (Set<String>) result.retData1;
Set<String> bookedGroupSet = broker.getOffsetManager().getBookedGroups();
if (inGroupNameSet.isEmpty()) {
qryGroupNameSet = bookedGroupSet;
@@ -717,19 +715,10 @@ public class BrokerAdminServlet extends AbstractWebHandler {
}
}
}
- // get the topic set to be queried
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, false, null);
- if (!result.success) {
- WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
- return;
- }
- // get target consume group name
- Set<String> topicSet = (Set<String>) result.retData1;
// verify the acquired Topic set and
// query the corresponding offset information
Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> groupOffsetMaps =
- getGroupOffsetInfo(qryGroupNameSet, topicSet);
+ getGroupOffsetInfo(WebFieldDef.COMPSGROUPNAME, qryGroupNameSet, topicSet);
// builder result
int totalCnt = 0;
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
@@ -772,26 +761,24 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminSetGroupOffSet(HttpServletRequest req,
StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
// get group list
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSGROUPNAME, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
Set<String> groupNameSet = (Set<String>) result.retData1;
// get set mode
- result = WebParameterUtils.getBooleanParamValue(req,
- WebFieldDef.MANUALSET, true, false);
- if (!result.success) {
+ if (!WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.MANUALSET, true, false, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
boolean manualSet = (Boolean) result.retData1;
// get modify user
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.MODIFYUSER, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -799,17 +786,15 @@ public class BrokerAdminServlet extends AbstractWebHandler {
final String modifier = (String) result.retData1;
if (manualSet) {
// get offset json info
- result = WebParameterUtils.getJsonDictParamValue(req,
- WebFieldDef.OFFSETJSON, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getJsonDictParamValue(req,
+ WebFieldDef.OFFSETJSON, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
Map<String, Long> manOffsets =
(Map<String, Long>) result.retData1;
// valid and transfer offset format
- result = validManOffsetResetInfo(WebFieldDef.OFFSETJSON, manOffsets);
- if (!result.success) {
+ if (!validManOffsetResetInfo(WebFieldDef.OFFSETJSON, manOffsets, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -817,9 +802,8 @@ public class BrokerAdminServlet extends AbstractWebHandler {
(List<Tuple3<String, Integer, Long>>) result.retData1;
} else {
// get the topic set to be set
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
@@ -840,38 +824,43 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminCloneGroupOffSet(HttpServletRequest req,
StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
// get source consume group name
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.SRCGROUPNAME, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.SRCGROUPNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
final String srcGroupName = (String) result.retData1;
- // get modify user
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.MODIFYUSER, true, null);
- if (!result.success) {
+ // get source consume group's topic set cloned to target group
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
- final String modifier = (String) result.retData1;
- // get source consume group's topic set cloned to target group
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, false, null);
- if (!result.success) {
+ Set<String> srcTopicNameSet = (Set<String>) result.retData1;
+ // valid topic and get topic's partitionIds
+ if (!validAndGetTopicPartInfo(srcGroupName,
+ WebFieldDef.SRCGROUPNAME, srcTopicNameSet, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
+ Map<String, Set<Integer>> topicPartMap =
+ (Map<String, Set<Integer>>) result.retData1;
// get target consume group name
- Set<String> srcTopicNameSet = (Set<String>) result.retData1;
- result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.TGTCOMPSGROUPNAME, true, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TGTCOMPSGROUPNAME, true, null, result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return;
}
Set<String> tgtGroupNameSet = (Set<String>) result.retData1;
+ // get modify user
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ final String modifier = (String) result.retData1;
// check sourceGroup if existed
Set<String> bookedGroups = broker.getOffsetManager().getBookedGroups();
if (!bookedGroups.contains(srcGroupName)) {
@@ -882,16 +871,6 @@ public class BrokerAdminServlet extends AbstractWebHandler {
.append(" has not been registered on this Broker!").toString());
return;
}
- // valid topic and get topic's partitionIds
- Map<String, Set<Integer>> topicPartMap =
- validAndGetPartitions(srcGroupName, srcTopicNameSet);
- if (topicPartMap.isEmpty()) {
- WebParameterUtils.buildFailResult(sBuilder,
- new StringBuilder(512).append("Parameter ")
- .append(WebFieldDef.SRCGROUPNAME.name).append(": not found ")
- .append(srcGroupName).append(" subscribed topic set!").toString());
- return;
- }
// query offset from source group
Map<String, Map<Integer, Tuple2<Long, Long>>> srcGroupOffsets =
broker.getOffsetManager().queryGroupOffset(srcGroupName, topicPartMap);
@@ -904,6 +883,56 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
}
+ /***
+ * Remove consume group offset.
+ *
+ * @param req
+ * @param sBuilder process result
+ */
+ public void adminRemoveGroupOffSet(HttpServletRequest req,
+ StringBuilder sBuilder) {
+ ProcessResult result = new ProcessResult();
+ // get consume group name
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, true, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ Set<String> groupNameSet = (Set<String>) result.retData1;
+ // get modify user
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ final String modifier = (String) result.retData1;
+ // get need removed offset's topic
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ // get target consume group name
+ Set<String> topicNameSet = (Set<String>) result.retData1;
+ // get set mode
+ if (!WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.ONLYMEM, false, false, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ boolean onlyMemory = (Boolean) result.retData1;
+ if (!validAndGetGroupTopicInfo(groupNameSet, topicNameSet, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ Map<String, Map<String, Set<Integer>>> groupTopicPartMap =
+ (Map<String, Map<String, Set<Integer>>>) result.retData1;
+ broker.getOffsetManager().deleteGroupOffset(
+ onlyMemory, groupTopicPartMap, modifier);
+ // builder return result
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
+ }
+
// build reset offset info
private List<Tuple3<String, Integer, Long>> buildOffsetResetInfo(
Map<String, Map<Integer, Tuple2<Long, Long>>> topicPartOffsetMap) {
@@ -950,8 +979,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
List<Tuple3<String, Integer, Long>> result = new ArrayList<>();
MessageStoreManager storeManager = broker.getStoreManager();
// get topic's partition set
- Map<String, Set<Integer>> topicPartMap =
- validAndGetPartitions(null, topicSet);
+ Map<String, Set<Integer>> topicPartMap = getTopicPartitions(topicSet);
// fill current topic's max offset value
for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
if (entry.getKey() == null
@@ -979,15 +1007,15 @@ public class BrokerAdminServlet extends AbstractWebHandler {
}
// build reset offset info
- private ProcessResult validManOffsetResetInfo(WebFieldDef fieldDef,
- Map<String, Long> manOffsetInfoMap) {
+ private boolean validManOffsetResetInfo(WebFieldDef fieldDef,
+ Map<String, Long> manOffsetInfoMap,
+ ProcessResult result) {
String brokerId;
String topicName;
String strPartId;
int partitionId;
long adjOffset;
MessageStore store = null;
- ProcessResult procResult = new ProcessResult();
MessageStoreManager storeManager = broker.getStoreManager();
List<Tuple3<String, Integer, Long>> offsetVals = new ArrayList<>();
String localBrokerId = String.valueOf(broker.getTubeConfig().getBrokerId());
@@ -1001,12 +1029,12 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// parse and check partitionKey value
String[] keyItems = entry.getKey().split(TokenConstants.ATTR_SEP);
if (keyItems.length != 3) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name).append("'s key invalid:")
.append(entry.getKey())
.append(" must be brokerId:topicName:partitionId !").toString());
- return procResult;
+ return result.success;
}
brokerId = keyItems[0].trim();
topicName = keyItems[1].trim();
@@ -1018,12 +1046,12 @@ public class BrokerAdminServlet extends AbstractWebHandler {
try {
partitionId = Integer.parseInt(strPartId);
} catch (NumberFormatException e) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name).append("'s key invalid:")
.append(entry.getKey())
.append("'s partitionId value not number!").toString());
- return procResult;
+ return result.success;
}
// check and adjust offset value
try {
@@ -1041,65 +1069,129 @@ public class BrokerAdminServlet extends AbstractWebHandler {
offsetVals.add(new Tuple3<>(topicName, partitionId, adjOffset));
}
if (offsetVals.isEmpty()) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append("'s value is invalid!").toString());
+ .append(fieldDef.name).append("'s value is invalid!").toString());
} else {
- procResult.setSuccResult(offsetVals);
+ result.setSuccResult(offsetVals);
}
- return procResult;
+ return result.success;
}
// builder group's offset info
private Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> getGroupOffsetInfo(
- Set<String> groupSet, Set<String> topicSet) {
- long curReadDataOffset = -2;
- long curDataLag = -2;
+ WebFieldDef groupFldDef, Set<String> groupSet, Set<String> topicSet) {
+ ProcessResult result = new ProcessResult();
Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> groupOffsetMaps = new HashMap<>();
for (String group : groupSet) {
Map<String, Map<Integer, GroupOffsetInfo>> topicOffsetRet = new HashMap<>();
// valid and get topic's partitionIds
- Map<String, Set<Integer>> topicPartMap = validAndGetPartitions(group, topicSet);
- // get topic's publish info
- Map<String, Map<Integer, TopicPubStoreInfo>> topicStorePubInfoMap =
- broker.getStoreManager().getTopicPublishInfos(topicPartMap.keySet());
- // get group's booked offset info
- Map<String, Map<Integer, Tuple2<Long, Long>>> groupOffsetMap =
- broker.getOffsetManager().queryGroupOffset(group, topicPartMap);
- // get offset info array
- for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
- String topic = entry.getKey();
- Map<Integer, GroupOffsetInfo> partOffsetRet = new HashMap<>();
- Map<Integer, TopicPubStoreInfo> storeInfoMap = topicStorePubInfoMap.get(topic);
- Map<Integer, Tuple2<Long, Long>> partBookedMap = groupOffsetMap.get(topic);
- for (Integer partitionId : entry.getValue()) {
- GroupOffsetInfo offsetInfo = new GroupOffsetInfo(partitionId);
- offsetInfo.setPartPubStoreInfo(storeInfoMap.get(partitionId));
- offsetInfo.setConsumeOffsetInfo(partBookedMap.get(partitionId));
- String queryKey = buildQueryID(group, topic, partitionId);
- ConsumerNodeInfo nodeInfo = broker.getConsumerNodeInfo(queryKey);
- if (nodeInfo != null) {
- offsetInfo.setConsumeDataOffsetInfo(nodeInfo.getLastDataRdOffset());
+ if (validAndGetTopicPartInfo(group, groupFldDef, topicSet, result)) {
+ Map<String, Set<Integer>> topicPartMap =
+ (Map<String, Set<Integer>>) result.retData1;
+ // get topic's publish info
+ Map<String, Map<Integer, TopicPubStoreInfo>> topicStorePubInfoMap =
+ broker.getStoreManager().getTopicPublishInfos(topicPartMap.keySet());
+ // get group's booked offset info
+ Map<String, Map<Integer, Tuple2<Long, Long>>> groupOffsetMap =
+ broker.getOffsetManager().queryGroupOffset(group, topicPartMap);
+ // get offset info array
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ String topic = entry.getKey();
+ Map<Integer, GroupOffsetInfo> partOffsetRet = new HashMap<>();
+ Map<Integer, TopicPubStoreInfo> storeInfoMap = topicStorePubInfoMap.get(topic);
+ Map<Integer, Tuple2<Long, Long>> partBookedMap = groupOffsetMap.get(topic);
+ for (Integer partitionId : entry.getValue()) {
+ GroupOffsetInfo offsetInfo = new GroupOffsetInfo(partitionId);
+ offsetInfo.setPartPubStoreInfo(
+ storeInfoMap == null ? null :storeInfoMap.get(partitionId));
+ offsetInfo.setConsumeOffsetInfo(
+ partBookedMap == null ? null : partBookedMap.get(partitionId));
+ String queryKey = buildQueryID(group, topic, partitionId);
+ ConsumerNodeInfo nodeInfo = broker.getConsumerNodeInfo(queryKey);
+ if (nodeInfo != null) {
+ offsetInfo.setConsumeDataOffsetInfo(nodeInfo.getLastDataRdOffset());
+ }
+ offsetInfo.calculateLag();
+ partOffsetRet.put(partitionId, offsetInfo);
}
- offsetInfo.calculateLag();
- partOffsetRet.put(partitionId, offsetInfo);
+ topicOffsetRet.put(topic, partOffsetRet);
}
- topicOffsetRet.put(topic, partOffsetRet);
}
groupOffsetMaps.put(group, topicOffsetRet);
}
return groupOffsetMaps;
}
+ // valid and get need removed group-topic info
+ private boolean validAndGetGroupTopicInfo(Set<String> groupSet,
+ Set<String> topicSet,
+ ProcessResult result) {
+ Map<String, Map<String, Set<Integer>>> groupTopicPartMap = new HashMap<>();
+ // filter group
+ Set<String> targetGroupSet = new HashSet<>();
+ Set<String> bookedGroups = broker.getOffsetManager().getBookedGroups();
+ for (String orgGroup : groupSet) {
+ if (bookedGroups.contains(orgGroup)) {
+ targetGroupSet.add(orgGroup);
+ }
+ }
+ // valid specified topic set
+ for (String group : targetGroupSet) {
+ if (validAndGetTopicPartInfo(group, WebFieldDef.GROUPNAME, topicSet, result)) {
+ Map<String, Set<Integer>> topicPartMap =
+ (Map<String, Set<Integer>>) result.retData1;
+ groupTopicPartMap.put(group, topicPartMap);
+ }
+ }
+ result.setSuccResult(groupTopicPartMap);
+ return true;
+ }
- private Map<String, Set<Integer>> validAndGetPartitions(String group, Set<String> topicSet) {
- Map<String, Set<Integer>> topicPartMap = new HashMap<>();
- // query stored topic set stored in memory or zk
- if (topicSet.isEmpty() && group != null) {
- topicSet = broker.getOffsetManager().getGroupSubInfo(group);
+ private boolean validAndGetTopicPartInfo(String groupName,
+ WebFieldDef groupFldDef,
+ Set<String> topicSet,
+ ProcessResult result) {
+ Set<String> subTopicSet =
+ broker.getOffsetManager().getGroupSubInfo(groupName);
+ if (subTopicSet == null || subTopicSet.isEmpty()) {
+ result.setFailResult(400, new StringBuilder(512)
+ .append("Parameter ").append(groupFldDef.name)
+ .append(": subscribed topic set of ").append(groupName)
+ .append(" query result is null!").toString());
+ return result.success;
}
- // get topic's partitionIds
+ // filter valid topic set
+ Set<String> tgtTopicSet = new HashSet<>();
+ if (topicSet.isEmpty()) {
+ tgtTopicSet = subTopicSet;
+ } else {
+ for (String topic : topicSet) {
+ if (subTopicSet.contains(topic)) {
+ tgtTopicSet.add(topic);
+ }
+ }
+ if (tgtTopicSet.isEmpty()) {
+ result.setFailResult(400, new StringBuilder(512)
+ .append("Parameter ").append(groupFldDef.name)
+ .append(": ").append(groupName)
+ .append(" unsubscribed to the specified topic set!").toString());
+ return result.success;
+ }
+ }
+ Map<String, Set<Integer>> topicPartMap = getTopicPartitions(tgtTopicSet);
+ if (topicPartMap.isEmpty()) {
+ result.setFailResult(400, new StringBuilder(512)
+ .append("Parameter ").append(groupFldDef.name)
+ .append(": all topics subscribed by the group have been deleted!").toString());
+ return result.success;
+ }
+ result.setSuccResult(topicPartMap);
+ return result.success;
+ }
+
+ private Map<String, Set<Integer>> getTopicPartitions(Set<String> topicSet) {
+ Map<String, Set<Integer>> topicPartMap = new HashMap<>();
if (topicSet != null) {
Map<String, TopicMetadata> topicConfigMap =
broker.getMetadataManager().getTopicConfigMap();
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index 45b862d..a65a223 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -81,8 +81,10 @@ public enum WebFieldDef {
TBaseConstants.META_MAX_GROUPNAME_LENGTH, RegexDef.TMP_GROUP),
MANUALSET(20, "manualSet", "manSet",
WebFieldType.BOOLEAN, "Whether manual offset setting mode"),
- OFFSETJSON(21, "offsetJsonSet", "offsetSet",
- WebFieldType.JSONTYPE, "The offset set that needs to be added or modified");
+ OFFSETJSON(21, "offsetJsonInfo", "offsetInfo",
+ WebFieldType.JSONTYPE, "The offset info that needs to be added or modified"),
+ ONLYMEM(22, "onlyMemory", "onlyMem", WebFieldType.BOOLEAN,
+ "Only clear the offset data in the memory cache, default is false");
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fileconfig/ZKConfig.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fileconfig/ZKConfig.java
index 5ac3683..8e677a5 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fileconfig/ZKConfig.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fileconfig/ZKConfig.java
@@ -28,7 +28,6 @@ public class ZKConfig {
private int zkSyncTimeMs = 1000;
private long zkCommitPeriodMs = 5000L;
private int zkCommitFailRetries = TServerConstants.CFG_ZK_COMMIT_DEFAULT_RETRIES;
-
public ZKConfig() {
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java
index dca7ca8..f5eee40 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java
@@ -18,6 +18,7 @@
package org.apache.tubemq.server.common.offsetstorage;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -33,10 +34,12 @@ public interface OffsetStorage {
final Collection<OffsetStorageInfo> offsetInfoList,
boolean isFailRetry);
- Map<String, Map<String, Set<String>>> getZkGroupTopicBrokerInfos();
+ Map<String, Set<String>> queryZkAllGroupTopicInfos();
- Map<String, Set<String>> getZkLocalGroupTopicInfos();
+ Map<String, Set<String>> queryZKGroupTopicInfo(List<String> groupSet);
Map<Integer, Long> queryGroupOffsetInfo(String group, String topic,
Set<Integer> partitionIds);
+
+ void deleteGroupOffsetInfo(Map<String, Map<String, Set<Integer>>> groupTopicPartMap);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java
index 8094151..3700753 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java
@@ -35,6 +35,8 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+
/**
* A offset storage implementation with zookeeper
*/
@@ -64,18 +66,16 @@ public class ZkOffsetStorage implements OffsetStorage {
private final String consumerZkDir;
private final boolean isBroker;
private final int brokerId;
+ private final String strBrokerId;
private ZKConfig zkConfig;
private ZooKeeperWatcher zkw;
- // group-topic-brokerid
- private final Map<String, Map<String, Set<String>>> zkGroupTopicBrokerInfos = new HashMap<>();
- // group-topic
- private final Map<String, Set<String>> zkLocalGroupTopicInfos = new HashMap<>();
public ZkOffsetStorage(final ZKConfig zkConfig, boolean isBroker, int brokerId) {
this.zkConfig = zkConfig;
- this.brokerId = brokerId;
this.isBroker = isBroker;
+ this.brokerId = brokerId;
+ this.strBrokerId = String.valueOf(brokerId);
this.tubeZkRoot = normalize(this.zkConfig.getZkNodeRoot());
this.consumerZkDir = this.tubeZkRoot + "/consumers-v3";
try {
@@ -86,8 +86,6 @@ public class ZkOffsetStorage implements OffsetStorage {
.append(this.zkConfig.getZkServerAddr()).append(") !").toString(), e);
System.exit(1);
}
- logger.info("[ZkOffsetStorage] Get group-topic-broker info from ZooKeeper");
- queryAllZKGroupTopicInfo();
logger.info("[ZkOffsetStorage] ZooKeeper Offset Storage initiated!");
}
@@ -102,16 +100,6 @@ public class ZkOffsetStorage implements OffsetStorage {
}
@Override
- public Map<String, Map<String, Set<String>>> getZkGroupTopicBrokerInfos() {
- return zkGroupTopicBrokerInfos;
- }
-
- @Override
- public Map<String, Set<String>> getZkLocalGroupTopicInfos() {
- return zkLocalGroupTopicInfos;
- }
-
- @Override
public void commitOffset(final String group,
final Collection<OffsetStorageInfo> offsetInfoList,
boolean isFailRetry) {
@@ -242,63 +230,130 @@ public class ZkOffsetStorage implements OffsetStorage {
}
/**
- * Get group-topic-brokerid map info stored in zookeeper.
- * <p/>
- * The broker only cares about the content of its own node,
- * so this part only queries when the node starts, and
- * caches relevant data in the memory for finding
- *
+ * Query booked topic info of groups stored in zookeeper.
+ * @param groupSet query groups
+ * @return group--topic map info
*/
- private void queryAllZKGroupTopicInfo() {
+ @Override
+ public Map<String, Set<String>> queryZKGroupTopicInfo(List<String> groupSet) {
+ String qryBrokerId;
+ Map<String, Set<String>> groupTopicMap = new HashMap<>();
StringBuilder sBuider = new StringBuilder(512);
- // get all booked groups name
+ if (groupSet == null || groupSet.isEmpty()) {
+ return groupTopicMap;
+ }
+ // build path base
String groupNode = sBuider.append(this.consumerZkDir).toString();
- List<String> bookedGroups = ZKUtil.getChildren(this.zkw, groupNode);
sBuider.delete(0, sBuider.length());
- if (bookedGroups != null) {
- // get topic info by group
- for (String group : bookedGroups) {
- String topicNode = sBuider.append(groupNode)
- .append("/").append(group).append("/offsets").toString();
- List<String> consumeTopics = ZKUtil.getChildren(this.zkw, topicNode);
- sBuider.delete(0, sBuider.length());
- Set<String> topicSet = new HashSet<>();
- Map<String, Set<String>> topicBrokerSet = new HashMap<>();
- if (consumeTopics != null) {
- // get broker info by topic
- for (String topic : consumeTopics) {
- String brokerNode = sBuider.append(topicNode)
- .append("/").append(topic).toString();
- List<String> brokerIds = ZKUtil.getChildren(this.zkw, brokerNode);
- sBuider.delete(0, sBuider.length());
- Set<String> brokerIdSet = new HashSet<>();
- if (brokerIds != null) {
- for (String idStr : brokerIds) {
- if (idStr != null) {
- String[] brokerPartIdStrs =
- idStr.split(TokenConstants.HYPHEN);
- brokerIdSet.add(brokerPartIdStrs[0]);
+ // get the group managed by this broker
+ for (String group : groupSet) {
+ String topicNode = sBuider.append(groupNode)
+ .append("/").append(group).append("/offsets").toString();
+ List<String> consumeTopics = ZKUtil.getChildren(this.zkw, topicNode);
+ sBuider.delete(0, sBuider.length());
+ Set<String> topicSet = new HashSet<>();
+ if (consumeTopics != null) {
+ for (String topic : consumeTopics) {
+ if (topic == null) {
+ continue;
+ }
+ String brokerNode = sBuider.append(topicNode)
+ .append("/").append(topic).toString();
+ List<String> brokerIds = ZKUtil.getChildren(this.zkw, brokerNode);
+ sBuider.delete(0, sBuider.length());
+ if (brokerIds != null) {
+ for (String idStr : brokerIds) {
+ if (idStr != null) {
+ String[] brokerPartIdStrs =
+ idStr.split(TokenConstants.HYPHEN);
+ qryBrokerId = brokerPartIdStrs[0];
+ if (qryBrokerId != null
+ && strBrokerId.equals(qryBrokerId.trim())) {
+ topicSet.add(topic);
+ break;
}
}
- if (isBroker && brokerIdSet.contains(String.valueOf(brokerId))) {
- topicSet.add(topic);
- }
}
- topicBrokerSet.put(topic, brokerIdSet);
}
}
- if (!topicSet.isEmpty()) {
- zkLocalGroupTopicInfos.put(group, topicSet);
+ }
+ if (!topicSet.isEmpty()) {
+ groupTopicMap.put(group, topicSet);
+ }
+ }
+ return groupTopicMap;
+ }
+
+ /**
+ * Get group-topic map info stored in zookeeper.
+ * <p/>
+ * The broker only cares about the content of its own node
+ *
+ */
+ @Override
+ public Map<String, Set<String>> queryZkAllGroupTopicInfos() {
+ StringBuilder sBuider = new StringBuilder(512);
+ // get all booked groups name
+ String groupNode = sBuider.append(this.consumerZkDir).toString();
+ List<String> bookedGroups = ZKUtil.getChildren(this.zkw, groupNode);
+ return queryZKGroupTopicInfo(bookedGroups);
+ }
+
+ /**
+ * Get offset stored in zookeeper, if not found or error, set null
+ * <p/>
+ *
+ * @return partitionId--offset map info
+ */
+ @Override
+ public void deleteGroupOffsetInfo(
+ Map<String, Map<String, Set<Integer>>> groupTopicPartMap) {
+ StringBuilder sBuider = new StringBuilder(512);
+ for (Map.Entry<String, Map<String, Set<Integer>>> entry
+ : groupTopicPartMap.entrySet()) {
+ if (entry.getKey() == null
+ || entry.getValue() == null
+ || entry.getValue().isEmpty()) {
+ continue;
+ }
+ String basePath = sBuider.append(this.consumerZkDir).append("/")
+ .append(entry.getKey()).append("/offsets").toString();
+ sBuider.delete(0, sBuider.length());
+ Map<String, Set<Integer>> topicPartMap = entry.getValue();
+ for (Map.Entry<String, Set<Integer>> topicEntry : topicPartMap.entrySet()) {
+ if (topicEntry.getKey() == null
+ || topicEntry.getValue() == null
+ || topicEntry.getValue().isEmpty()) {
+ continue;
+ }
+ Set<Integer> partIdSet = topicEntry.getValue();
+ for (Integer partitionId : partIdSet) {
+ String offsetNode = sBuider.append(basePath).append("/")
+ .append(topicEntry.getKey()).append("/")
+ .append(brokerId).append(TokenConstants.HYPHEN)
+ .append(partitionId).toString();
+ sBuider.delete(0, sBuider.length());
+ ZKUtil.delZNode(this.zkw, offsetNode);
}
- zkGroupTopicBrokerInfos.put(group, topicBrokerSet);
+ String parentNode = sBuider.append(basePath).append("/")
+ .append(topicEntry.getKey()).toString();
+ sBuider.delete(0, sBuider.length());
+ chkAndRmvBlankParentNode(parentNode);
}
+ chkAndRmvBlankParentNode(basePath);
+ String parentNode = sBuider.append(this.consumerZkDir)
+ .append("/").append(entry.getKey()).toString();
+ sBuider.delete(0, sBuider.length());
+ chkAndRmvBlankParentNode(parentNode);
}
- logger.info(new StringBuilder(256)
- .append("[ZkOffsetStorage] query from zookeeper, total group size = ")
- .append(zkGroupTopicBrokerInfos.size()).append(", local group size = ")
- .append(zkLocalGroupTopicInfos.size()).toString());
}
+ private void chkAndRmvBlankParentNode(String parentNode) {
+ List<String> nodeSet = ZKUtil.getChildren(zkw, parentNode);
+ if (nodeSet != null && nodeSet.isEmpty()) {
+ ZKUtil.delZNode(this.zkw, parentNode);
+ }
+ }
private String normalize(final String root) {
if (root.startsWith("/")) {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java
index da86191..5eb6cea 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java
@@ -167,6 +167,20 @@ public class ZKUtil {
}
}
+ /**
+ * delete the specified znode.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ */
+ public static void delZNode(ZooKeeperWatcher zkw, String znode) {
+ try {
+ zkw.getRecoverableZooKeeper().delete(znode, -1);
+ } catch (Throwable e) {
+ //
+ }
+ }
+
private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
boolean watcherSet) throws KeeperException {
try {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
index 3688b1c..8cabf67 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
@@ -44,10 +44,13 @@ public class ProcessResult {
this.success = false;
this.errCode = errCode;
this.errInfo = errMsg;
+ this.retData1 = null;
}
public void setSuccResult(Object retData) {
this.success = true;
+ this.errInfo = "";
+ this.errCode = TErrCodeConstants.SUCCESS;
this.retData1 = retData;
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index fddd5de..2318bdb 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -249,32 +249,32 @@ public class WebParameterUtils {
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
* @param defValue a default value returned if failed to parse value from the given object
- * @return valid result for the parameter value
+ * @param result process result of parameter value
+ * @return process result
*/
- public static ProcessResult getLongParamValue(HttpServletRequest req,
- WebFieldDef fieldDef,
- boolean required,
- long defValue) {
- ProcessResult procResult =
- getStringParamValue(req, fieldDef, required, null);
- if (!procResult.success) {
- return procResult;
- }
- String paramValue = (String) procResult.retData1;
+ public static boolean getLongParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ long defValue,
+ ProcessResult result) {
+ if (!getStringParamValue(req, fieldDef, required, null, result)) {
+ return result.success;
+ }
+ String paramValue = (String) result.retData1;
if (paramValue == null) {
- procResult.setSuccResult(defValue);
- return procResult;
+ result.setSuccResult(defValue);
+ return result.success;
}
try {
long paramIntVal = Long.parseLong(paramValue);
- procResult.setSuccResult(paramIntVal);
+ result.setSuccResult(paramIntVal);
} catch (Throwable e) {
- procResult.setFailResult(400,
+ result.setFailResult(400,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name).append(" parse error: ")
.append(e.getMessage()).toString());
}
- return procResult;
+ return result.success;
}
/**
@@ -283,43 +283,44 @@ public class WebParameterUtils {
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
- * @return valid result for the parameter value
+ * @param result process result of parameter value
+ * @return process result
*/
- public static ProcessResult getIntParamValue(HttpServletRequest req,
- WebFieldDef fieldDef,
- boolean required) {
- ProcessResult procResult =
- getStringParamValue(req, fieldDef, required, null);
- if (!procResult.success) {
- return procResult;
+ public static boolean getIntParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ ProcessResult result) {
+ if (!getStringParamValue(req, fieldDef,
+ required, null, result)) {
+ return result.success;
}
- ProcessResult procRet = new ProcessResult();
Set<Integer> tgtValueSet = new HashSet<Integer>();
if (fieldDef.isCompFieldType()) {
- Set<String> valItemSet = (Set<String>) procResult.retData1;
+ Set<String> valItemSet = (Set<String>) result.retData1;
if (valItemSet.isEmpty()) {
- procResult.setSuccResult(tgtValueSet);
- return procResult;
+ result.setSuccResult(tgtValueSet);
+ return result.success;
}
for (String itemVal : valItemSet) {
- if (!checkIntValueNorms(procRet, fieldDef, itemVal, false, -1)) {
- return procRet;
+ if (!checkIntValueNorms(fieldDef,
+ itemVal, false, -1, result)) {
+ return result.success;
}
- tgtValueSet.add((Integer) procRet.retData1);
+ tgtValueSet.add((Integer) result.retData1);
}
} else {
- String paramValue = (String) procResult.retData1;
+ String paramValue = (String) result.retData1;
if (paramValue == null) {
- procResult.setSuccResult(tgtValueSet);
- return procResult;
+ result.setSuccResult(tgtValueSet);
+ return result.success;
}
- if (!checkIntValueNorms(procRet,
- fieldDef, paramValue, false, -1)) {
- tgtValueSet.add((Integer) procRet.retData1);
+ if (!checkIntValueNorms(fieldDef,
+ paramValue, false, -1, result)) {
+ tgtValueSet.add((Integer) result.retData1);
}
}
- procResult.setSuccResult(tgtValueSet);
- return procResult;
+ result.setSuccResult(tgtValueSet);
+ return result.success;
}
/**
@@ -330,43 +331,44 @@ public class WebParameterUtils {
* @param required a boolean value represent whether the parameter is must required
* @param defValue a default value returned if failed to parse value from the given object
* @param minValue min value required
- * @return valid result for the parameter value
+ * @param result process result of parameter value
+ * @return process result
*/
- public static ProcessResult getIntParamValue(HttpServletRequest req,
+ public static boolean getIntParamValue(HttpServletRequest req,
WebFieldDef fieldDef,
boolean required,
int defValue,
- int minValue) {
- ProcessResult procResult =
- getStringParamValue(req, fieldDef, required, null);
- if (!procResult.success) {
- return procResult;
+ int minValue,
+ ProcessResult result) {
+ if (!getStringParamValue(req, fieldDef, required, null, result)) {
+ return result.success;
}
if (fieldDef.isCompFieldType()) {
Set<Integer> tgtValueSet = new HashSet<Integer>();
- Set<String> valItemSet = (Set<String>) procResult.retData1;
+ Set<String> valItemSet = (Set<String>) result.retData1;
if (valItemSet.isEmpty()) {
tgtValueSet.add(defValue);
- procResult.setSuccResult(tgtValueSet);
- return procResult;
+ result.setSuccResult(tgtValueSet);
+ return result.success;
}
- ProcessResult procRet = new ProcessResult();
for (String itemVal : valItemSet) {
- if (!checkIntValueNorms(procRet, fieldDef, itemVal, true, minValue)) {
- return procRet;
+ if (!checkIntValueNorms(fieldDef,
+ itemVal, true, minValue, result)) {
+ return result.success;
}
- tgtValueSet.add((Integer) procRet.retData1);
+ tgtValueSet.add((Integer) result.retData1);
}
- procResult.setSuccResult(tgtValueSet);
+ result.setSuccResult(tgtValueSet);
} else {
- String paramValue = (String) procResult.retData1;
+ String paramValue = (String) result.retData1;
if (paramValue == null) {
- procResult.setSuccResult(defValue);
- return procResult;
+ result.setSuccResult(defValue);
+ return result.success;
}
- checkIntValueNorms(procResult, fieldDef, paramValue, true, minValue);
+ checkIntValueNorms(fieldDef,
+ paramValue, true, minValue, result);
}
- return procResult;
+ return result.success;
}
/**
@@ -376,24 +378,24 @@ public class WebParameterUtils {
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
* @param defValue a default value returned if failed to parse value from the given object
+ * @param result process result
* @return valid result for the parameter value
*/
- public static ProcessResult getBooleanParamValue(HttpServletRequest req,
- WebFieldDef fieldDef,
- boolean required,
- boolean defValue) {
- ProcessResult procResult =
- getStringParamValue(req, fieldDef, required, null);
- if (!procResult.success) {
- return procResult;
- }
- String paramValue = (String) procResult.retData1;
+ public static boolean getBooleanParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ boolean defValue,
+ ProcessResult result) {
+ if (!getStringParamValue(req, fieldDef, required, null, result)) {
+ return result.success;
+ }
+ String paramValue = (String) result.retData1;
if (paramValue == null) {
- procResult.setSuccResult(defValue);
- return procResult;
+ result.setSuccResult(defValue);
+ return result.success;
}
- procResult.setSuccResult(Boolean.parseBoolean(paramValue));
- return procResult;
+ result.setSuccResult(Boolean.parseBoolean(paramValue));
+ return result.success;
}
/**
@@ -403,13 +405,14 @@ public class WebParameterUtils {
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
* @param defValue a default value returned if failed to parse value from the given object
+ * @param result process result
* @return valid result for the parameter value
*/
- public static ProcessResult getStringParamValue(HttpServletRequest req,
- WebFieldDef fieldDef,
- boolean required,
- String defValue) {
- ProcessResult procResult = new ProcessResult();
+ public static boolean getStringParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ String defValue,
+ ProcessResult result) {
// get parameter value
String paramValue = req.getParameter(fieldDef.name);
if (paramValue == null) {
@@ -422,14 +425,14 @@ public class WebParameterUtils {
// Check if the parameter exists
if (TStringUtils.isBlank(paramValue)) {
if (required) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append(" is missing or value is null or blank!").toString());
} else {
- procStringDefValue(procResult, fieldDef.isCompFieldType(), defValue);
+ procStringDefValue(fieldDef.isCompFieldType(), defValue, result);
}
- return procResult;
+ return result.success;
}
// check if value is norm;
if (fieldDef.isCompFieldType()) {
@@ -440,41 +443,41 @@ public class WebParameterUtils {
if (TStringUtils.isBlank(strParamValueItem)) {
continue;
}
- if (!checkStrValueNorms(procResult, fieldDef, strParamValueItem)) {
- return procResult;
+ if (!checkStrValueNorms(fieldDef, strParamValueItem, result)) {
+ return result.success;
}
- valItemSet.add((String) procResult.retData1);
+ valItemSet.add((String) result.retData1);
}
// check if is empty result
if (valItemSet.isEmpty()) {
if (required) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append(" is missing or value is null or blank!").toString());
} else {
- procStringDefValue(procResult, fieldDef.isCompFieldType(), defValue);
+ procStringDefValue(fieldDef.isCompFieldType(), defValue, result);
}
- return procResult;
+ return result.success;
}
// check max item count
if (fieldDef.itemMaxCnt != TBaseConstants.META_VALUE_UNDEFINED) {
if (valItemSet.size() > fieldDef.itemMaxCnt) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append("'s item count over max allowed count (")
.append(fieldDef.itemMaxCnt).append(")!").toString());
}
}
- procResult.setSuccResult(valItemSet);
+ result.setSuccResult(valItemSet);
} else {
- if (!checkStrValueNorms(procResult, fieldDef, paramValue)) {
- return procResult;
+ if (!checkStrValueNorms(fieldDef, paramValue, result)) {
+ return result.success;
}
- procResult.setSuccResult(paramValue);
+ result.setSuccResult(paramValue);
}
- return procResult;
+ return result.success;
}
/**
@@ -484,13 +487,14 @@ public class WebParameterUtils {
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
* @param defValue a default value returned if failed to parse value from the given object
+ * @param result process result
* @return valid result for the parameter value
*/
- public static ProcessResult getJsonDictParamValue(HttpServletRequest req,
- WebFieldDef fieldDef,
- boolean required,
- Map<String, Long> defValue) {
- ProcessResult procResult = new ProcessResult();
+ public static boolean getJsonDictParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ Map<String, Long> defValue,
+ ProcessResult result) {
// get parameter value
String paramValue = req.getParameter(fieldDef.name);
if (paramValue == null) {
@@ -503,20 +507,20 @@ public class WebParameterUtils {
// Check if the parameter exists
if (TStringUtils.isBlank(paramValue)) {
if (required) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append(" is missing or value is null or blank!").toString());
} else {
- procResult.setSuccResult(defValue);
+ result.setSuccResult(defValue);
}
- return procResult;
+ return result.success;
}
try {
paramValue = URLDecoder.decode(paramValue,
TBaseConstants.META_DEFAULT_CHARSET_NAME);
} catch (UnsupportedEncodingException e) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append(" decode error, exception is ")
@@ -524,73 +528,83 @@ public class WebParameterUtils {
}
if (TStringUtils.isBlank(paramValue)) {
if (required) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append("'s value is blank!").toString());
} else {
- procResult.setSuccResult(defValue);
+ result.setSuccResult(defValue);
}
- return procResult;
+ return result.success;
}
if (fieldDef.valMaxLen != TBaseConstants.META_VALUE_UNDEFINED) {
if (paramValue.length() > fieldDef.valMaxLen) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name)
.append("'s length over max allowed length (")
.append(fieldDef.valMaxLen).append(")!").toString());
- return procResult;
+ return result.success;
}
}
- procResult.setSuccResult(new Gson().fromJson(paramValue,
- new TypeToken<Map<String, Long>>(){}.getType()));
- return procResult;
+ // parse data
+ try {
+ Map<String, Long> manOffsets = new Gson().fromJson(paramValue,
+ new TypeToken<Map<String, Long>>(){}.getType());
+ result.setSuccResult(manOffsets);
+ } catch (Throwable e) {
+ result.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append(" value parse failure, error is ")
+ .append(e.getMessage()).append("!").toString());
+ }
+ return result.success;
}
/**
* process string default value
*
- * @param procResult process result
* @param isCompFieldType the parameter if compound field type
* @param defValue the parameter default value
+ * @param result process result
* @return process result for default value of parameter
*/
- private static ProcessResult procStringDefValue(ProcessResult procResult,
- boolean isCompFieldType,
- String defValue) {
+ private static boolean procStringDefValue(boolean isCompFieldType,
+ String defValue,
+ ProcessResult result) {
if (isCompFieldType) {
Set<String> valItemSet = new HashSet<>();
if (TStringUtils.isNotBlank(defValue)) {
valItemSet.add(defValue);
}
- procResult.setSuccResult(valItemSet);
+ result.setSuccResult(valItemSet);
} else {
- procResult.setSuccResult(defValue);
+ result.setSuccResult(defValue);
}
- return procResult;
+ return result.success;
}
/**
* Parse the parameter string value by regex define
*
- * @param procResult process result
* @param fieldDef the parameter field definition
* @param paramVal the parameter value
+ * @param result process result
* @return check result for string value of parameter
*/
- private static boolean checkStrValueNorms(ProcessResult procResult,
- WebFieldDef fieldDef,
- String paramVal) {
+ private static boolean checkStrValueNorms(WebFieldDef fieldDef,
+ String paramVal,
+ ProcessResult result) {
paramVal = paramVal.trim();
if (TStringUtils.isBlank(paramVal)) {
- procResult.setSuccResult(null);
+ result.setSuccResult(null);
return true;
}
// check value's max length
if (fieldDef.valMaxLen != TBaseConstants.META_VALUE_UNDEFINED) {
if (paramVal.length() > fieldDef.valMaxLen) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("over max length for ")
.append(fieldDef.name).append(", only allow ")
.append(fieldDef.valMaxLen).append(" length").toString());
@@ -600,44 +614,44 @@ public class WebParameterUtils {
// check value's pattern
if (fieldDef.regexCheck) {
if (!paramVal.matches(fieldDef.regexDef.getPattern())) {
- procResult.setFailResult(fieldDef.id,
+ result.setFailResult(fieldDef.id,
new StringBuilder(512).append("illegal value for ")
.append(fieldDef.name).append(", value ")
.append(fieldDef.regexDef.getErrMsgTemp()).toString());
return false;
}
}
- procResult.setSuccResult(paramVal);
+ result.setSuccResult(paramVal);
return true;
}
/**
* Parse the parameter string value by regex define
*
- * @param procResult process result
* @param fieldDef the parameter field definition
* @param paramValue the parameter value
* @param hasMinVal whether there is a minimum
* param minValue the parameter min value
+ * @param result process result
* @return check result for string value of parameter
*/
- private static boolean checkIntValueNorms(ProcessResult procResult,
- WebFieldDef fieldDef,
+ private static boolean checkIntValueNorms(WebFieldDef fieldDef,
String paramValue,
boolean hasMinVal,
- int minValue) {
+ int minValue,
+ ProcessResult result) {
try {
int paramIntVal = Integer.parseInt(paramValue);
if (hasMinVal && paramIntVal < minValue) {
- procResult.setFailResult(400,
+ result.setFailResult(400,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name).append(" value must >= ")
.append(minValue).toString());
return false;
}
- procResult.setSuccResult(paramIntVal);
+ result.setSuccResult(paramIntVal);
} catch (Throwable e) {
- procResult.setFailResult(400,
+ result.setFailResult(400,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name).append(" parse error: ")
.append(e.getMessage()).toString());
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
index 3bdfe16..8bef5ab 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
@@ -207,7 +207,7 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
WebParameterUtils.validIntDataParameter("memCacheMsgSizeInMB",
req.getParameter("memCacheMsgSizeInMB"),
false, defmemCacheMsgSizeInMB, 2);
- memCacheMsgSizeInMB = memCacheMsgSizeInMB >= 2048 ? 2048 : memCacheMsgSizeInMB;
+ memCacheMsgSizeInMB = Math.min(memCacheMsgSizeInMB, 2048);
int memCacheFlushIntvl =
WebParameterUtils.validIntDataParameter("memCacheFlushIntvl",
req.getParameter("memCacheFlushIntvl"),
@@ -354,7 +354,7 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
WebParameterUtils.validIntDataParameter("memCacheMsgSizeInMB",
jsonObject.get("memCacheMsgSizeInMB"),
false, brokerConfEntity.getDftMemCacheMsgSizeInMB(), 2);
- memCacheMsgSizeInMB = memCacheMsgSizeInMB >= 2048 ? 2048 : memCacheMsgSizeInMB;
+ memCacheMsgSizeInMB = Math.min(memCacheMsgSizeInMB, 2048);
int memCacheFlushIntvl =
WebParameterUtils.validIntDataParameter("memCacheFlushIntvl",
jsonObject.get("memCacheFlushIntvl"),
@@ -714,10 +714,10 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
* @return
*/
public StringBuilder adminQuerySimpleTopicName(HttpServletRequest req) {
+ ProcessResult result = new ProcessResult();
StringBuilder strBuffer = new StringBuilder(512);
- ProcessResult result = WebParameterUtils.getIntParamValue(req,
- WebFieldDef.COMPSBROKERID, false);
- if (!result.success) {
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.COMPSBROKERID, false, result)) {
WebParameterUtils.buildFailResult(strBuffer, result.errInfo);
return strBuffer;
}
@@ -752,17 +752,16 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
* @return
*/
public StringBuilder adminQuerySimpleBrokerId(HttpServletRequest req) {
+ ProcessResult result = new ProcessResult();
StringBuilder strBuffer = new StringBuilder(512);
- ProcessResult result = WebParameterUtils.getStringParamValue(req,
- WebFieldDef.COMPSTOPICNAME, false, null);
- if (!result.success) {
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
WebParameterUtils.buildFailResult(strBuffer, result.errInfo);
return strBuffer;
}
Set<String> topicNameSet = (Set<String>) result.retData1;
- result = WebParameterUtils.getBooleanParamValue(req,
- WebFieldDef.WITHIP, false, false);
- if (!result.success) {
+ if (!WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.WITHIP, false, false, result)) {
WebParameterUtils.buildFailResult(strBuffer, result.errInfo);
return strBuffer;
}
[incubator-tubemq] 07/49: [TUBEMQ-437] Fix tubemq table source sink
factory instance creating problem (#342)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 86ccc70a0251112f108b45a181cf2255d7929ac9
Author: duli559 <55...@qq.com>
AuthorDate: Fri Dec 4 19:53:16 2020 +0800
[TUBEMQ-437] Fix tubemq table source sink factory instance creating problem (#342)
Co-authored-by: herodu <he...@tencent.com>
---
.../apache/flink/connectors/tubemq/TubemqTableSourceSinkFactory.java | 2 +-
.../META-INF/services/org.apache.flink.table.factories.TableFactory | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqTableSourceSinkFactory.java b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqTableSourceSinkFactory.java
index fa006ad..4b581d6 100644
--- a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqTableSourceSinkFactory.java
+++ b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqTableSourceSinkFactory.java
@@ -69,7 +69,7 @@ public class TubemqTableSourceSinkFactory implements StreamTableSourceFactory<Ro
private static final String SPLIT_COMMA = ",";
- private TubemqTableSourceSinkFactory() {
+ public TubemqTableSourceSinkFactory() {
}
@Override
diff --git a/tubemq-connectors/tubemq-connector-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory b/tubemq-connectors/tubemq-connector-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory
index 61b040d..3083174 100644
--- a/tubemq-connectors/tubemq-connector-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory
+++ b/tubemq-connectors/tubemq-connector-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.flink.connectors.tubemq.TubemqTableSourceFactory
+org.apache.flink.connectors.tubemq.TubemqTableSourceSinkFactory
[incubator-tubemq] 17/49: [TUBEMQ-451]Replace ConsumeTupleInfo with
Tuple2 (#349)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 9e184b1049f940b6b1955bd35271119d42dcad54
Author: gosonzhang <46...@qq.com>
AuthorDate: Tue Dec 15 14:08:54 2020 +0800
[TUBEMQ-451]Replace ConsumeTupleInfo with Tuple2 (#349)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../org/apache/tubemq/server/master/TMaster.java | 37 +++++++++++-----------
.../nodeconsumer/ConsumerInfoHolder.java | 14 ++------
2 files changed, 22 insertions(+), 29 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
index b1bf1dc..8811cb0 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
@@ -71,6 +71,7 @@ import org.apache.tubemq.corebase.utils.ConcurrentHashSet;
import org.apache.tubemq.corebase.utils.DataConverterUtil;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corebase.utils.ThreadUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.corerpc.RpcConfig;
import org.apache.tubemq.corerpc.RpcConstants;
import org.apache.tubemq.corerpc.RpcServiceFactory;
@@ -1883,14 +1884,14 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if (consumerId == null) {
continue;
}
- ConsumerInfoHolder.ConsumeTupleInfo tupleInfo =
+ Tuple2<String, ConsumerInfo> tupleInfo =
consumerHolder.getConsumeTupleInfo(consumerId);
if (tupleInfo == null
- || tupleInfo.groupName == null
- || tupleInfo.consumerInfo == null) {
+ || tupleInfo.f0 == null
+ || tupleInfo.f1 == null) {
continue;
}
- List<String> blackTopicList = this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.groupName);
+ List<String> blackTopicList = this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
Map<String, List<Partition>> topicSubPartMap = entry.getValue();
List<SubscribeInfo> deletedSubInfoList = new ArrayList<>();
List<SubscribeInfo> addedSubInfoList = new ArrayList<>();
@@ -1907,7 +1908,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
currentPartMap = new HashMap<>();
}
}
- if (tupleInfo.consumerInfo.isOverTLS()) {
+ if (tupleInfo.f1.isOverTLS()) {
for (Partition currentPart : currentPartMap.values()) {
if (!blackTopicList.contains(currentPart.getTopic())) {
boolean found = false;
@@ -1923,8 +1924,8 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
}
}
deletedSubInfoList
- .add(new SubscribeInfo(consumerId, tupleInfo.groupName,
- tupleInfo.consumerInfo.isOverTLS(), currentPart));
+ .add(new SubscribeInfo(consumerId, tupleInfo.f0,
+ tupleInfo.f1.isOverTLS(), currentPart));
}
for (Partition finalPart : finalPartList) {
if (!blackTopicList.contains(finalPart.getTopic())) {
@@ -1940,7 +1941,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
continue;
}
addedSubInfoList.add(new SubscribeInfo(consumerId,
- tupleInfo.groupName, true, finalPart));
+ tupleInfo.f0, true, finalPart));
}
}
} else {
@@ -1948,14 +1949,14 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if ((blackTopicList.contains(currentPart.getTopic()))
|| (!finalPartList.contains(currentPart))) {
deletedSubInfoList
- .add(new SubscribeInfo(consumerId, tupleInfo.groupName, false, currentPart));
+ .add(new SubscribeInfo(consumerId, tupleInfo.f0, false, currentPart));
}
}
for (Partition finalPart : finalPartList) {
if ((currentPartMap.get(finalPart.getPartitionKey()) == null)
&& (!blackTopicList.contains(finalPart.getTopic()))) {
addedSubInfoList.add(new SubscribeInfo(consumerId,
- tupleInfo.groupName, false, finalPart));
+ tupleInfo.f0, false, finalPart));
}
}
}
@@ -2033,16 +2034,16 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if (consumerId == null) {
continue;
}
- ConsumerInfoHolder.ConsumeTupleInfo tupleInfo =
+ Tuple2<String, ConsumerInfo> tupleInfo =
consumerHolder.getConsumeTupleInfo(consumerId);
if (tupleInfo == null
- || tupleInfo.groupName == null
- || tupleInfo.consumerInfo == null) {
+ || tupleInfo.f0 == null
+ || tupleInfo.f1 == null) {
continue;
}
List<String> blackTopicList =
- this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.groupName);
+ this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
Map<String, Map<String, Partition>> topicSubPartMap = entry.getValue();
List<SubscribeInfo> deletedSubInfoList = new ArrayList<>();
List<SubscribeInfo> addedSubInfoList = new ArrayList<>();
@@ -2066,15 +2067,15 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if ((blackTopicList.contains(currentPart.getTopic()))
|| (finalPartMap.get(currentPart.getPartitionKey()) == null)) {
deletedSubInfoList
- .add(new SubscribeInfo(consumerId, tupleInfo.groupName,
- tupleInfo.consumerInfo.isOverTLS(), currentPart));
+ .add(new SubscribeInfo(consumerId, tupleInfo.f0,
+ tupleInfo.f1.isOverTLS(), currentPart));
}
}
for (Partition finalPart : finalPartMap.values()) {
if ((currentPartMap.get(finalPart.getPartitionKey()) == null)
&& (!blackTopicList.contains(finalPart.getTopic()))) {
- addedSubInfoList.add(new SubscribeInfo(consumerId, tupleInfo.groupName,
- tupleInfo.consumerInfo.isOverTLS(), finalPart));
+ addedSubInfoList.add(new SubscribeInfo(consumerId, tupleInfo.f0,
+ tupleInfo.f1.isOverTLS(), finalPart));
}
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodeconsumer/ConsumerInfoHolder.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodeconsumer/ConsumerInfoHolder.java
index 5c78858..fc5d932 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodeconsumer/ConsumerInfoHolder.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodeconsumer/ConsumerInfoHolder.java
@@ -26,6 +26,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.tubemq.corebase.cluster.ConsumerInfo;
+import org.apache.tubemq.corebase.utils.Tuple2;
public class ConsumerInfoHolder {
@@ -369,7 +370,7 @@ public class ConsumerInfoHolder {
return consumer;
}
- public ConsumeTupleInfo getConsumeTupleInfo(String consumerId) {
+ public Tuple2<String, ConsumerInfo> getConsumeTupleInfo(String consumerId) {
try {
rwLock.readLock().lock();
ConsumerInfo consumerInfo = null;
@@ -378,7 +379,7 @@ public class ConsumerInfoHolder {
if (consumeBandInfo != null) {
consumerInfo = consumeBandInfo.getConsumerInfo(consumerId);
}
- return new ConsumeTupleInfo(groupName, consumerInfo);
+ return new Tuple2<String, ConsumerInfo>(groupName, consumerInfo);
} finally {
rwLock.readLock().unlock();
}
@@ -424,13 +425,4 @@ public class ConsumerInfoHolder {
groupInfoMap.clear();
}
- public class ConsumeTupleInfo {
- public String groupName;
- public ConsumerInfo consumerInfo;
-
- public ConsumeTupleInfo(String groupName, ConsumerInfo consumerInfo) {
- this.groupName = groupName;
- this.consumerInfo = consumerInfo;
- }
- }
}
[incubator-tubemq] 45/49: [TUBEMQ-529] Update CHANGE.md
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 3fcbb38942e9c30e198110fe5b6b78e9932fd186
Author: yuanbo <yu...@apache.org>
AuthorDate: Tue Jan 19 14:18:44 2021 +0800
[TUBEMQ-529] Update CHANGE.md
---
CHANGES.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGES.md b/CHANGES.md
index 977b738..3e14289 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -56,6 +56,7 @@
| [TUBEMQ-511](https://issues.apache.org/jira/browse/TUBEMQ-511) | Replace the conditional operator (?:) with mid() | Major |
| [TUBEMQ-512](https://issues.apache.org/jira/browse/TUBEMQ-512) | Add package length control based on Topic | Major |
| [TUBEMQ-515](https://issues.apache.org/jira/browse/TUBEMQ-515) | Add cluster Topic view web api | Major |
+| [TUBEMQ-526](https://issues.apache.org/jira/browse/TUBEMQ-526) | Adjust the packaging script and version check list, remove the "-WIP" tag | Major |
### BUG FIXES:
| JIRA | Summary | Priority |
[incubator-tubemq] 23/49: [TUBEMQ-484]Add query API for topic
publication information
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 868b04c4ae18bb800864fba142e4a116826b4a6c
Author: gosonzhang <go...@tencent.com>
AuthorDate: Fri Dec 25 18:48:58 2020 +0800
[TUBEMQ-484]Add query API for topic publication information
---
.../server/broker/utils/GroupOffsetInfo.java | 8 ++--
.../server/broker/utils/TopicPubStoreInfo.java | 28 ++++++++----
.../server/broker/web/BrokerAdminServlet.java | 51 +++++++++++++++++++++-
3 files changed, 72 insertions(+), 15 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
index 9a4abe3..a0c7215 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
@@ -39,10 +39,10 @@ public class GroupOffsetInfo {
public void setPartPubStoreInfo(TopicPubStoreInfo pubStoreInfo) {
if (pubStoreInfo != null) {
- this.offsetMin = pubStoreInfo.indexStart;
- this.offsetMax = pubStoreInfo.indexEnd;
- this.dataMin = pubStoreInfo.dataStart;
- this.dataMax = pubStoreInfo.dataEnd;
+ this.offsetMin = pubStoreInfo.offsetMin;
+ this.offsetMax = pubStoreInfo.offsetMax;
+ this.dataMin = pubStoreInfo.dataMin;
+ this.dataMax = pubStoreInfo.dataMax;
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java
index b2257dd..73b9d69 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java
@@ -26,20 +26,30 @@ public class TopicPubStoreInfo {
public String topicName = null;
public int storeId = TBaseConstants.META_VALUE_UNDEFINED;
public int partitionId = TBaseConstants.META_VALUE_UNDEFINED;
- public long indexStart = 0L;
- public long indexEnd = 0L;
- public long dataStart = 0L;
- public long dataEnd = 0L;
+ public long offsetMin = 0L;
+ public long offsetMax = 0L;
+ public long dataMin = 0L;
+ public long dataMax = 0L;
public TopicPubStoreInfo(String topicName, int storeId, int partitionId,
- long indexStart, long indexEnd, long dataStart, long dataEnd) {
+ long offsetMin, long offsetMax, long dataMin, long dataMax) {
this.topicName = topicName;
this.storeId = storeId;
this.partitionId = partitionId;
- this.indexStart = indexStart;
- this.indexEnd = indexEnd;
- this.dataStart = dataStart;
- this.dataEnd = dataEnd;
+ this.offsetMin = offsetMin;
+ this.offsetMax = offsetMax;
+ this.dataMin = dataMin;
+ this.dataMax = dataMax;
+ }
+
+ public StringBuilder buildPubStoreInfo(StringBuilder sBuilder) {
+ sBuilder.append("{\"partitionId\":").append(partitionId)
+ .append(",\"offsetMin\":").append(offsetMin)
+ .append(",\"offsetMax\":").append(offsetMax)
+ .append(",\"dataMin\":").append(dataMin)
+ .append(",\"dataMax\":").append(dataMax)
+ .append("}");
+ return sBuilder;
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index 9a0506e..d8f85d4 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -77,6 +77,9 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// get all registered methods
innRegisterWebMethod("admin_get_methods",
"adminQueryAllMethods");
+ // query topic's publish info
+ innRegisterWebMethod("admin_query_pubinfo",
+ "adminQueryPubInfo");
// Query all consumer groups booked on the Broker.
innRegisterWebMethod("admin_query_group",
"adminQueryBookedGroup");
@@ -467,7 +470,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
* @throws Exception
*/
public void adminQueryCurrentGroupOffSet(HttpServletRequest req,
- StringBuilder sBuilder) throws Exception {
+ StringBuilder sBuilder) {
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.TOPICNAME, true, null);
if (!result.success) {
@@ -576,6 +579,50 @@ public class BrokerAdminServlet extends AbstractWebHandler {
}
/***
+ * Query topic's publish info on the Broker.
+ *
+ * @param req
+ * @param sBuilder process result
+ */
+ public void adminQueryPubInfo(HttpServletRequest req,
+ StringBuilder sBuilder) {
+ // get the topic set to be queried
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ // get target consume group name
+ Set<String> topicSet = (Set<String>) result.retData1;
+ // get topic's publish info
+ Map<String, Map<Integer, TopicPubStoreInfo>> topicStorePubInfoMap =
+ broker.getStoreManager().getTopicPublishInfos(topicSet);
+ // builder result
+ int totalCnt = 0;
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
+ for (Map.Entry<String, Map<Integer, TopicPubStoreInfo>> entry
+ : topicStorePubInfoMap.entrySet()) {
+ if (totalCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("{\"topicName\":\"").append(entry.getKey())
+ .append("\",\"offsetInfo\":[");
+ Map<Integer, TopicPubStoreInfo> storeInfoMap = entry.getValue();
+ int itemCnt = 0;
+ for (Map.Entry<Integer, TopicPubStoreInfo> entry1 : storeInfoMap.entrySet()) {
+ if (itemCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ TopicPubStoreInfo pubStoreInfo = entry1.getValue();
+ pubStoreInfo.buildPubStoreInfo(sBuilder);
+ }
+ sBuilder.append("],\"itemCount\":").append(itemCnt).append("}");
+ }
+ sBuilder.append("],\"dataCount\":").append(totalCnt).append("}");
+ }
+
+ /***
* Query all consumer groups booked on the Broker.
*
* @param req
@@ -583,7 +630,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
*/
public void adminQueryBookedGroup(HttpServletRequest req,
StringBuilder sBuilder) {
- // get group list
+ // get divide info
ProcessResult result = WebParameterUtils.getBooleanParamValue(req,
WebFieldDef.WITHDIVIDE, false, false);
if (!result.success) {
[incubator-tubemq] 36/49: [TUBEMQ-510] Found a bug in
MessageProducerExample class
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit a7ae74e164b551baca51ef5104d2a9653a694fe4
Author: gosonzhang <go...@tencent.com>
AuthorDate: Tue Jan 12 14:55:23 2021 +0800
[TUBEMQ-510] Found a bug in MessageProducerExample class
---
.../src/main/java/org/apache/tubemq/example/MessageProducerExample.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
index 2b806b5..0bac5ef 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
@@ -104,7 +104,7 @@ public final class MessageProducerExample {
while (msgCount < 0 || sentCount < msgCount) {
roundIndex = (int) (sentCount++ % targetCnt);
Tuple2<String, String> target = topicSendRounds.get(roundIndex);
- Message message = new Message(target.getF0(), body.getBytes());
+ Message message = new Message(target.getF0(), dataBuffer.array());
long currTimeMillis = System.currentTimeMillis();
message.setAttrKeyVal("index", String.valueOf(sentCount));
message.setAttrKeyVal("dataTime", String.valueOf(currTimeMillis));
[incubator-tubemq] 33/49: [TUBEMQ-508] Optimize Broker's PB
parameter check processing logic (#392)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit fb22584e6ff821e395499ea83393dc82d14b2932
Author: gosonzhang <46...@qq.com>
AuthorDate: Tue Jan 12 16:54:04 2021 +0800
[TUBEMQ-508] Optimize Broker's PB parameter check processing logic (#392)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../tubemq/server/broker/BrokerServiceServer.java | 184 +++++++++++----------
.../tubemq/server/common/fielddef/WebFieldDef.java | 6 +-
.../server/common/paramcheck/PBParameterUtils.java | 160 +++++++++---------
3 files changed, 181 insertions(+), 169 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
index 60490c6..c9cfba4 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
@@ -69,13 +69,14 @@ import org.apache.tubemq.server.common.TStatusConstants;
import org.apache.tubemq.server.common.aaaserver.CertificateBrokerHandler;
import org.apache.tubemq.server.common.aaaserver.CertifiedResult;
import org.apache.tubemq.server.common.exception.HeartbeatException;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
import org.apache.tubemq.server.common.heartbeat.HeartbeatManager;
import org.apache.tubemq.server.common.heartbeat.TimeoutInfo;
import org.apache.tubemq.server.common.heartbeat.TimeoutListener;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorageInfo;
import org.apache.tubemq.server.common.paramcheck.PBParameterUtils;
-import org.apache.tubemq.server.common.paramcheck.ParamCheckResult;
import org.apache.tubemq.server.common.utils.AppendResult;
+import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.common.utils.RowLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -271,6 +272,8 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
public GetMessageResponseB2C getMessagesC2B(GetMessageRequestC2B request,
final String rmtAddress,
boolean overtls) throws Throwable {
+ ProcessResult result = new ProcessResult();
+ StringBuilder strBuffer = new StringBuilder(512);
final GetMessageResponseB2C.Builder builder =
GetMessageResponseB2C.newBuilder();
builder.setSuccess(false);
@@ -278,39 +281,37 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setEscFlowCtrl(false);
builder.setCurrDataDlt(-1);
builder.setMinLimitTime(0);
- StringBuilder strBuffer = new StringBuilder(512);
if (!this.started.get()
|| ServiceStatusHolder.isReadServiceStop()) {
builder.setErrCode(TErrCodeConstants.SERVICE_UNAVAILABLE);
builder.setErrMsg("Read StoreService temporary unavailable!");
return builder.build();
}
- // check request's parameters
- ParamCheckResult paramCheckResult =
- PBParameterUtils.checkClientId(request.getClientId(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get and check clientId field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.CLIENTID,
+ request.getClientId(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String clientId = (String) paramCheckResult.checkData;
- paramCheckResult =
- PBParameterUtils.checkGroupName(request.getGroupName(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ final String clientId = (String) result.retData1;
+ // get and check groupName field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.GROUPNAME,
+ request.getGroupName(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String groupName = (String) paramCheckResult.checkData;
- paramCheckResult =
- PBParameterUtils.checkConsumeTopicName(request.getTopicName(), this.metadataManager, strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ final String groupName = (String) result.retData1;
+ // get and check topicName field
+ if (!PBParameterUtils.getTopicNameParameter(request.getTopicName(),
+ this.metadataManager, strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
// get consumer info
- final String topicName = (String) paramCheckResult.checkData;
+ final String topicName = (String) result.retData1;
final int partitionId = request.getPartitionId();
boolean isEscFlowCtrl = request.hasEscFlowCtrl() && request.getEscFlowCtrl();
String partStr = getPartStr(groupName, topicName, partitionId);
@@ -587,6 +588,7 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
public SendMessageResponseB2P sendMessageP2B(SendMessageRequestP2B request,
final String rmtAddress,
boolean overtls) throws Throwable {
+ ProcessResult result = new ProcessResult();
final StringBuilder strBuffer = new StringBuilder(512);
SendMessageResponseB2P.Builder builder = SendMessageResponseB2P.newBuilder();
builder.setSuccess(false);
@@ -603,24 +605,23 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg(certResult.errInfo);
return builder.build();
}
- ParamCheckResult paramCheckResult =
- PBParameterUtils.checkClientId(request.getClientId(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get and check clientId field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.CLIENTID,
+ request.getClientId(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String producerId = (String) paramCheckResult.checkData;
- paramCheckResult =
- PBParameterUtils.checkExistTopicNameInfo(request.getTopicName(),
- request.getPartitionId(), this.metadataManager, strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ final String producerId = (String) result.retData1;
+ // get and check topicName and partitionId field
+ final int partitionId = request.getPartitionId();
+ if (!PBParameterUtils.getTopicNamePartIdInfo(request.getTopicName(),
+ partitionId, this.metadataManager, strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String reqTopic = (String) paramCheckResult.checkData;
- final int partition = request.getPartitionId();
+ final String topicName = (String) result.retData1;
String msgType = null;
int msgTypeCode = -1;
if (TStringUtils.isNotBlank(request.getMsgType())) {
@@ -645,14 +646,14 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
if (request.getCheckSum() != -1 && checkSum != request.getCheckSum()) {
builder.setErrCode(TErrCodeConstants.FORBIDDEN);
builder.setErrMsg(strBuffer.append("Checksum msg data failure: ")
- .append(request.getCheckSum()).append(" of ").append(reqTopic)
+ .append(request.getCheckSum()).append(" of ").append(topicName)
.append(" not equal to the data's checksum of ")
.append(checkSum).toString());
return builder.build();
}
CertifiedResult authorizeResult =
serverAuthHandler.validProduceAuthorizeInfo(
- certResult.userName, reqTopic, msgType, rmtAddress);
+ certResult.userName, topicName, msgType, rmtAddress);
if (!authorizeResult.result) {
builder.setErrCode(authorizeResult.errCode);
builder.setErrMsg(authorizeResult.errInfo);
@@ -660,11 +661,11 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
}
try {
final MessageStore store =
- this.storeManager.getOrCreateMessageStore(reqTopic, partition);
+ this.storeManager.getOrCreateMessageStore(topicName, partitionId);
final AppendResult appendResult = new AppendResult();
if (store.appendMsg(appendResult, dataLength, checkSum, msgData,
- msgTypeCode, request.getFlag(), partition, request.getSentAddr())) {
- String baseKey = strBuffer.append(reqTopic)
+ msgTypeCode, request.getFlag(), partitionId, request.getSentAddr())) {
+ String baseKey = strBuffer.append(topicName)
.append("#").append(AddressUtils.intToIp(request.getSentAddr()))
.append("#").append(tubeConfig.getHostName())
.append("#").append(request.getPartitionId())
@@ -712,6 +713,7 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
public RegisterResponseB2C consumerRegisterC2B(RegisterRequestC2B request,
final String rmtAddress,
boolean overtls) throws Throwable {
+ ProcessResult result = new ProcessResult();
final StringBuilder strBuffer = new StringBuilder(512);
RegisterResponseB2C.Builder builder = RegisterResponseB2C.newBuilder();
builder.setSuccess(false);
@@ -727,28 +729,31 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg(certResult.errInfo);
return builder.build();
}
- ParamCheckResult paramCheckResult = PBParameterUtils.checkClientId(request.getClientId(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get and check clientId field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.CLIENTID,
+ request.getClientId(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String clientId = (String) paramCheckResult.checkData;
- paramCheckResult
- = PBParameterUtils.checkConsumeTopicName(request.getTopicName(), this.metadataManager, strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ final String clientId = (String) result.retData1;
+ // get and check topicName field
+ if (!PBParameterUtils.getTopicNameParameter(request.getTopicName(),
+ this.metadataManager, strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String topicName = (String) paramCheckResult.checkData;
- paramCheckResult = PBParameterUtils.checkGroupName(request.getGroupName(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get consumer info
+ final String topicName = (String) result.retData1;
+ // get and check groupName field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.GROUPNAME,
+ request.getGroupName(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String groupName = (String) paramCheckResult.checkData;
+ final String groupName = (String) result.retData1;
boolean isRegister = (request.getOpType() == RpcConstants.MSG_OPTYPE_REGISTER);
Set<String> filterCondSet = new HashSet<>();
if (request.getFilterCondStrList() != null && !request.getFilterCondStrList().isEmpty()) {
@@ -977,8 +982,9 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
public HeartBeatResponseB2C consumerHeartbeatC2B(HeartBeatRequestC2B request,
final String rmtAddress,
boolean overtls) throws Throwable {
- final HeartBeatResponseB2C.Builder builder = HeartBeatResponseB2C.newBuilder();
+ ProcessResult result = new ProcessResult();
final StringBuilder strBuffer = new StringBuilder(512);
+ final HeartBeatResponseB2C.Builder builder = HeartBeatResponseB2C.newBuilder();
builder.setSuccess(false);
if (!this.started.get()) {
builder.setErrCode(TErrCodeConstants.SERVICE_UNAVAILABLE);
@@ -992,22 +998,22 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg(certResult.errInfo);
return builder.build();
}
- ParamCheckResult paramCheckResult =
- PBParameterUtils.checkClientId(request.getClientId(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get and check clientId field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.CLIENTID,
+ request.getClientId(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String clientId = (String) paramCheckResult.checkData;
- paramCheckResult =
- PBParameterUtils.checkGroupName(request.getGroupName(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ final String clientId = (String) result.retData1;
+ // get and check groupName field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.GROUPNAME,
+ request.getGroupName(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String groupName = (String) paramCheckResult.checkData;
+ final String groupName = (String) result.retData1;
int reqQryPriorityId = request.hasQryPriorityId()
? request.getQryPriorityId() : TBaseConstants.META_VALUE_UNDEFINED;
List<Partition> partitions =
@@ -1097,8 +1103,9 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
public CommitOffsetResponseB2C consumerCommitC2B(CommitOffsetRequestC2B request,
final String rmtAddress,
boolean overtls) throws Throwable {
- final CommitOffsetResponseB2C.Builder builder = CommitOffsetResponseB2C.newBuilder();
+ ProcessResult result = new ProcessResult();
StringBuilder strBuffer = new StringBuilder(512);
+ final CommitOffsetResponseB2C.Builder builder = CommitOffsetResponseB2C.newBuilder();
builder.setSuccess(false);
builder.setCurrOffset(-1);
if (!this.started.get()) {
@@ -1106,32 +1113,31 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg("StoreService temporary unavailable!");
return builder.build();
}
- ParamCheckResult paramCheckResult =
- PBParameterUtils.checkClientId(request.getClientId(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get and check clientId field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.CLIENTID,
+ request.getClientId(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String clientId = (String) paramCheckResult.checkData;
- paramCheckResult =
- PBParameterUtils.checkGroupName(request.getGroupName(), strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ final String clientId = (String) result.retData1;
+ // get and check groupName field
+ if (!PBParameterUtils.getStringParameter(WebFieldDef.GROUPNAME,
+ request.getGroupName(), strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String groupName = (String) paramCheckResult.checkData;
+ final String groupName = (String) result.retData1;
int partitionId = request.getPartitionId();
- paramCheckResult =
- PBParameterUtils.checkExistTopicNameInfo(
- request.getTopicName(), partitionId, this.metadataManager, strBuffer);
- if (!paramCheckResult.result) {
- builder.setErrCode(paramCheckResult.errCode);
- builder.setErrMsg(paramCheckResult.errMsg);
+ // get and check topicName and partitionId field
+ if (!PBParameterUtils.getTopicNamePartIdInfo(request.getTopicName(),
+ partitionId, this.metadataManager, strBuffer, result)) {
+ builder.setErrCode(result.errCode);
+ builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String topicName = (String) paramCheckResult.checkData;
+ final String topicName = (String) result.retData1;
String partStr = getPartStr(groupName, topicName, partitionId);
ConsumerNodeInfo consumerNodeInfo = consumerRegisterMap.get(partStr);
if (consumerNodeInfo == null) {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index 2ed75c3..68a833b 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -93,7 +93,11 @@ public enum WebFieldDef {
CREATEDATE(25, "createDate", "cDate", WebFieldType.STRING,
"Record creation date", TBaseConstants.META_MAX_DATEVALUE_LENGTH),
MODIFYDATE(26, "modifyDate", "mDate", WebFieldType.STRING,
- "Record modification date", TBaseConstants.META_MAX_DATEVALUE_LENGTH);
+ "Record modification date", TBaseConstants.META_MAX_DATEVALUE_LENGTH),
+ HOSTNAME(27, "hostName", "hostName", WebFieldType.STRING,
+ "Host name information", TBaseConstants.META_MAX_CLIENT_HOSTNAME_LENGTH),
+ CLIENTID(28, "clientId", "clientId", WebFieldType.STRING,
+ "Client ID", TBaseConstants.META_MAX_CLIENT_ID_LENGTH);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
index ed2e122..1c2d5ca 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
@@ -42,6 +42,9 @@ import org.apache.tubemq.server.master.nodemanage.nodeconsumer.ConsumerBandInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+
+
public class PBParameterUtils {
private static final Logger logger = LoggerFactory.getLogger(PBParameterUtils.class);
@@ -438,85 +441,6 @@ public class PBParameterUtils {
}
/**
- * Check the topic name.
- *
- * @param topicName the topic name to check
- * @param metadataManager the metadata manager which contains topic information
- * @param strBuffer the string buffer used to construct the check result
- * @return the check result
- */
- public static ParamCheckResult checkConsumeTopicName(final String topicName,
- final MetadataManager metadataManager,
- final StringBuilder strBuffer) {
- ParamCheckResult retResult = new ParamCheckResult();
- if (TStringUtils.isBlank(topicName)) {
- retResult.setCheckResult(false,
- TErrCodeConstants.BAD_REQUEST,
- "Request miss necessary topicName data!");
- return retResult;
- }
- String tmpValue = topicName.trim();
- if (metadataManager.getTopicMetadata(tmpValue) == null) {
- retResult.setCheckResult(false,
- TErrCodeConstants.FORBIDDEN,
- strBuffer.append("Topic ").append(tmpValue)
- .append(" not existed, please check your configure").toString());
- return retResult;
- }
- retResult.setCheckData(tmpValue);
- return retResult;
- }
-
- /**
- * Check the existing topic name info
- *
- * @param topicName the topic name to be checked.
- * @param partitionId the partition ID where the topic locates
- * @param metadataManager the metadata manager which contains topic information
- * @param strBuffer the string buffer used to construct the check result
- * @return the check result
- */
- public static ParamCheckResult checkExistTopicNameInfo(final String topicName,
- final int partitionId,
- final MetadataManager metadataManager,
- final StringBuilder strBuffer) {
- ParamCheckResult retResult = new ParamCheckResult();
- if (TStringUtils.isBlank(topicName)) {
- retResult.setCheckResult(false,
- TErrCodeConstants.BAD_REQUEST,
- "Request miss necessary topicName data!");
- return retResult;
- }
- String tmpValue = topicName.trim();
- TopicMetadata topicMetadata = metadataManager.getTopicMetadata(tmpValue);
- if (topicMetadata == null) {
- retResult.setCheckResult(false,
- TErrCodeConstants.FORBIDDEN,
- strBuffer.append("Topic ").append(tmpValue)
- .append(" not existed, please check your configure").toString());
- return retResult;
- }
- if (metadataManager.isClosedTopic(tmpValue)) {
- retResult.setCheckResult(false,
- TErrCodeConstants.FORBIDDEN,
- strBuffer.append("Topic ").append(tmpValue).append(" has been closed").toString());
- return retResult;
- }
- int realPartition = partitionId < TBaseConstants.META_STORE_INS_BASE
- ? partitionId : partitionId % TBaseConstants.META_STORE_INS_BASE;
- if ((realPartition < 0) || (realPartition >= topicMetadata.getNumPartitions())) {
- retResult.setCheckResult(false,
- TErrCodeConstants.FORBIDDEN,
- strBuffer.append("Partition ")
- .append(tmpValue).append("-").append(partitionId)
- .append(" not existed, please check your configure").toString());
- return retResult;
- }
- retResult.setCheckData(tmpValue);
- return retResult;
- }
-
- /**
* Check the clientID.
*
* @param clientId the client id to be checked
@@ -609,4 +533,82 @@ public class PBParameterUtils {
result.setSuccResult(tmpValue);
return result.success;
}
+
+ /**
+ * Check the topic name.
+ *
+ * @param topicName the topic name to check
+ * @param metadataManager the metadata manager which contains topic information
+ * @param strBuffer the string buffer used to construct the check result
+ * @param result the checked result
+ * @return the check result
+ */
+ public static boolean getTopicNameParameter(String topicName,
+ MetadataManager metadataManager,
+ StringBuilder strBuffer,
+ ProcessResult result) {
+ if (!getStringParameter(WebFieldDef.TOPICNAME,
+ topicName, strBuffer, result)) {
+ return result.success;
+ }
+ String tmpValue = (String) result.retData1;
+ if (metadataManager.getTopicMetadata(tmpValue) == null) {
+ result.setFailResult(TErrCodeConstants.FORBIDDEN,
+ strBuffer.append(WebFieldDef.TOPICNAME.name)
+ .append(" ").append(tmpValue)
+ .append(" not existed, please check your configure").toString());
+ strBuffer.delete(0, strBuffer.length());
+ }
+ return result.success;
+ }
+
+ /**
+ * Check the existing topic name info
+ *
+ * @param topicName the topic name to be checked.
+ * @param partitionId the partition ID where the topic locates
+ * @param metadataManager the metadata manager which contains topic information
+ * @param strBuffer the string buffer used to construct the check result
+ * @param result the checked result
+ * @return the check result
+ */
+ public static boolean getTopicNamePartIdInfo(String topicName, int partitionId,
+ MetadataManager metadataManager,
+ StringBuilder strBuffer,
+ ProcessResult result) {
+ if (!getStringParameter(WebFieldDef.TOPICNAME,
+ topicName, strBuffer, result)) {
+ return result.success;
+ }
+ String tmpValue = (String) result.retData1;
+ TopicMetadata topicMetadata = metadataManager.getTopicMetadata(tmpValue);
+ if (topicMetadata == null) {
+ result.setFailResult(TErrCodeConstants.FORBIDDEN,
+ strBuffer.append(WebFieldDef.TOPICNAME.name)
+ .append(" ").append(tmpValue)
+ .append(" not existed, please check your configure").toString());
+ strBuffer.delete(0, strBuffer.length());
+ return result.success;
+ }
+ if (metadataManager.isClosedTopic(tmpValue)) {
+ result.setFailResult(TErrCodeConstants.FORBIDDEN,
+ strBuffer.append(WebFieldDef.TOPICNAME.name)
+ .append(" ").append(tmpValue)
+ .append(" has been closed").toString());
+ strBuffer.delete(0, strBuffer.length());
+ return result.success;
+ }
+ int realPartition = partitionId < TBaseConstants.META_STORE_INS_BASE
+ ? partitionId : partitionId % TBaseConstants.META_STORE_INS_BASE;
+ if ((realPartition < 0) || (realPartition >= topicMetadata.getNumPartitions())) {
+ result.setFailResult(TErrCodeConstants.FORBIDDEN,
+ strBuffer.append(WebFieldDef.PARTITIONID.name)
+ .append(" ").append(tmpValue).append("-").append(partitionId)
+ .append(" not existed, please check your configure").toString());
+ strBuffer.delete(0, strBuffer.length());
+ return result.success;
+ }
+ result.setSuccResult(tmpValue);
+ return result.success;
+ }
}
[incubator-tubemq] 32/49: [TUBEMQ-501] Adjust max message size
check logic
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 6ca161c8603d2cd98a5cb217ecb87f672682ef0d
Author: gosonzhang <go...@tencent.com>
AuthorDate: Mon Jan 11 16:18:46 2021 +0800
[TUBEMQ-501] Adjust max message size check logic
---
.../tubemq/client/producer/AllowedSetting.java | 4 +-
.../tubemq/client/producer/ProducerManager.java | 48 +++-
.../client/producer/SimpleMessageProducer.java | 25 +-
.../org/apache/tubemq/corebase/TBaseConstants.java | 18 +-
.../apache/tubemq/corebase/utils/MixedUtils.java | 10 +
.../tubemq/corebase/utils/SettingValidUtils.java | 19 +-
.../org/apache/tubemq/corerpc/RpcConstants.java | 6 +-
tubemq-core/src/main/proto/MasterService.proto | 10 +-
.../tubemq/server/broker/BrokerServiceServer.java | 5 +-
.../apache/tubemq/server/broker/TubeBroker.java | 270 ++++++++++++---------
.../broker/metadata/ClusterConfigHolder.java | 82 +++++++
.../server/broker/msgstore/MessageStore.java | 16 +-
.../server/broker/msgstore/disk/FileSegment.java | 5 +-
.../server/broker/msgstore/disk/MsgFileStore.java | 3 +-
.../server/broker/msgstore/mem/MsgMemStore.java | 3 +-
.../tubemq/server/common/fielddef/WebFieldDef.java | 4 +-
.../server/common/paramcheck/PBParameterUtils.java | 33 +++
.../server/common/utils/WebParameterUtils.java | 109 +++++----
.../org/apache/tubemq/server/master/TMaster.java | 72 ++++++
.../bdbentitys/BdbClusterSettingEntity.java | 172 +++++++------
.../master/web/handler/WebMasterInfoHandler.java | 13 +-
21 files changed, 650 insertions(+), 277 deletions(-)
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java
index cabf928..7beb4e0 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java
@@ -45,8 +45,8 @@ public class AllowedSetting {
}
if (allowedConfig.hasMaxMsgSize()
&& allowedConfig.getMaxMsgSize() != maxMsgSize.get()) {
- maxMsgSize.set(
- SettingValidUtils.validAndGetMaxMsgSize(allowedConfig.getMaxMsgSize()));
+ maxMsgSize.set(SettingValidUtils.validAndGetMaxMsgSizeInB(
+ allowedConfig.getMaxMsgSize()));
}
}
}
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/ProducerManager.java b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/ProducerManager.java
index ea1fd51..07586ec 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/ProducerManager.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/ProducerManager.java
@@ -79,6 +79,8 @@ public class ProducerManager {
private final ScheduledExecutorService heartbeatService;
private final AtomicLong visitToken =
new AtomicLong(TBaseConstants.META_VALUE_UNDEFINED);
+ private final AllowedSetting allowedSetting =
+ new AllowedSetting();
private final AtomicReference<String> authAuthorizedTokenRef =
new AtomicReference<>("");
private final ClientAuthenticateHandler authenticateHandler =
@@ -311,6 +313,15 @@ public class ProducerManager {
}
/**
+ * Get allowed message size.
+ *
+ * @return max allowed message size
+ */
+ public int getMaxMsgSize() {
+ return allowedSetting.getMaxMsgSize();
+ }
+
+ /**
* Check if the producer manager is shutdown.
*
* @return producer status
@@ -396,7 +407,7 @@ public class ProducerManager {
updateBrokerInfoList(true, response.getBrokerInfosList(),
response.getBrokerCheckSum(), sBuilder);
}
- processRegAuthorizedToken(response);
+ processRegSyncInfo(response);
return;
}
if (remainingRetry <= 0) {
@@ -436,6 +447,7 @@ public class ProducerManager {
if (authInfoBuilder != null) {
builder.setAuthInfo(authInfoBuilder.build());
}
+ builder.setAppdConfig(buildAllowedConfig4P());
return builder.build();
}
@@ -455,6 +467,7 @@ public class ProducerManager {
if (authInfoBuilder != null) {
builder.setAuthInfo(authInfoBuilder.build());
}
+ builder.setAppdConfig(buildAllowedConfig4P());
return builder.build();
}
@@ -544,13 +557,22 @@ public class ProducerManager {
}
}
- private void processRegAuthorizedToken(ClientMaster.RegisterResponseM2P response) {
+ private void processRegSyncInfo(ClientMaster.RegisterResponseM2P response) {
if (response.hasAuthorizedInfo()) {
processAuthorizedToken(response.getAuthorizedInfo());
}
+ if (response.hasAppdConfig()) {
+ procAllowedConfig4P(response.getAppdConfig());
+ }
}
- private void processHeartBeatAuthorizedToken(ClientMaster.HeartResponseM2P response) {
+ private void processHeartBeatSyncInfo(ClientMaster.HeartResponseM2P response) {
+ if (response.hasRequireAuth()) {
+ nextWithAuthInfo2M.set(response.getRequireAuth());
+ }
+ if (response.hasAppdConfig()) {
+ procAllowedConfig4P(response.getAppdConfig());
+ }
if (response.hasAuthorizedInfo()) {
processAuthorizedToken(response.getAuthorizedInfo());
}
@@ -595,6 +617,21 @@ public class ProducerManager {
return authInfoBuilder;
}
+ // build allowed configure info
+ private ClientMaster.ApprovedClientConfig.Builder buildAllowedConfig4P() {
+ ClientMaster.ApprovedClientConfig.Builder appdConfig =
+ ClientMaster.ApprovedClientConfig.newBuilder();
+ appdConfig.setConfigId(allowedSetting.getConfigId());
+ return appdConfig;
+ }
+
+ // set allowed configure info
+ private void procAllowedConfig4P(ClientMaster.ApprovedClientConfig allowedConfig) {
+ if (allowedConfig != null) {
+ allowedSetting.updAllowedSetting(allowedConfig);
+ }
+ }
+
// #lizard forgives
private class ProducerHeartbeatTask implements Runnable {
@Override
@@ -633,10 +670,7 @@ public class ProducerManager {
}
return;
}
- if (response.hasRequireAuth()) {
- nextWithAuthInfo2M.set(response.getRequireAuth());
- }
- processHeartBeatAuthorizedToken(response);
+ processHeartBeatSyncInfo(response);
if (response.getErrCode() == TErrCodeConstants.NOT_READY) {
lastHeartbeatTime = System.currentTimeMillis();
return;
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/SimpleMessageProducer.java b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/SimpleMessageProducer.java
index 7d6894d..946f03e 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/SimpleMessageProducer.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/SimpleMessageProducer.java
@@ -288,19 +288,6 @@ public class SimpleMessageProducer implements MessageProducer {
|| (message.getData().length == 0)) {
throw new TubeClientException("Illegal parameter: null data in message package!");
}
- int msgSize = TStringUtils.isBlank(message.getAttribute())
- ? message.getData().length : (message.getData().length + message.getAttribute().length());
- if (msgSize > TBaseConstants.META_MAX_MESSAGE_DATA_SIZE) {
- throw new TubeClientException(new StringBuilder(512)
- .append("Illegal parameter: over max message length for the total size of")
- .append(" message data and attribute, allowed size is ")
- .append(TBaseConstants.META_MAX_MESSAGE_DATA_SIZE)
- .append(", message's real size is ").append(msgSize).toString());
- }
- if (isShutDown.get()) {
- throw new TubeClientException("Status error: producer has been shutdown!");
- }
-
if (this.publishTopicMap.get(message.getTopic()) == null) {
throw new TubeClientException(new StringBuilder(512)
.append("Topic ").append(message.getTopic())
@@ -311,6 +298,18 @@ public class SimpleMessageProducer implements MessageProducer {
.append("Topic ").append(message.getTopic())
.append(" not publish, make sure the topic exist or acceptPublish and try later!").toString());
}
+ int msgSize = TStringUtils.isBlank(message.getAttribute())
+ ? message.getData().length : (message.getData().length + message.getAttribute().length());
+ if (msgSize > producerManager.getMaxMsgSize()) {
+ throw new TubeClientException(new StringBuilder(512)
+ .append("Illegal parameter: over max message length for the total size of")
+ .append(" message data and attribute, allowed size is ")
+ .append(producerManager.getMaxMsgSize())
+ .append(", message's real size is ").append(msgSize).toString());
+ }
+ if (isShutDown.get()) {
+ throw new TubeClientException("Status error: producer has been shutdown!");
+ }
}
private ClientBroker.SendMessageRequestP2B createSendMessageRequest(Partition partition,
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java
index d91b083..5238323 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java
@@ -29,9 +29,6 @@ public class TBaseConstants {
public static final String META_DEFAULT_CHARSET_NAME = "UTF-8";
public static final int META_MAX_MSGTYPE_LENGTH = 255;
- public static final int META_MAX_MESSAGE_HEADER_SIZE = 1024;
- public static final int META_MAX_MESSAGE_DATA_SIZE = 1024 * 1024;
- public static final int META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT = 20 * 1024 * 1024;
public static final int META_MAX_PARTITION_COUNT = 100;
public static final int META_MAX_BROKER_IP_LENGTH = 32;
public static final int META_MAX_USERNAME_LENGTH = 64;
@@ -67,4 +64,19 @@ public class TBaseConstants {
public static final long CFG_DEFAULT_AUTH_TIMESTAMP_VALID_INTERVAL = 20000;
+ public static final int META_MB_UNIT_SIZE = (1024 * 1024);
+ public static final int META_MESSAGE_SIZE_ADJUST = (512 * 1024);
+ public static final int META_MAX_MESSAGE_HEADER_SIZE = (10 * 1024);
+
+ public static final int META_MIN_ALLOWED_MESSAGE_SIZE_MB = 1;
+ public static final int META_MAX_ALLOWED_MESSAGE_SIZE_MB = 20;
+ public static final int META_MAX_MESSAGE_DATA_SIZE =
+ META_MIN_ALLOWED_MESSAGE_SIZE_MB * META_MB_UNIT_SIZE;
+ public static final int META_MIN_MEM_BUFFER_SIZE =
+ META_MAX_MESSAGE_DATA_SIZE + META_MESSAGE_SIZE_ADJUST;
+ public static final int META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT =
+ META_MAX_ALLOWED_MESSAGE_SIZE_MB * META_MB_UNIT_SIZE;
+
+
+
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
index bfbedad..1374b72 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
@@ -92,4 +92,14 @@ public class MixedUtils {
dataBuffer.flip();
return dataBuffer.array();
}
+
+ // get the middle data between min, max, and data
+ public static int mid(int data, int min, int max) {
+ return Math.max(min, Math.min(max, data));
+ }
+
+ public static long mid(long data, long min, long max) {
+ return Math.max(min, Math.min(max, data));
+ }
+
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java
index 4a206ef..e748ba4 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java
@@ -22,18 +22,17 @@ import org.apache.tubemq.corebase.TBaseConstants;
public class SettingValidUtils {
- // get the middle data between min, max, and data
- public static int mid(int data, int min, int max) {
- return Math.max(min, Math.min(max, data));
- }
- public static long mid(long data, long min, long max) {
- return Math.max(min, Math.min(max, data));
+ public static int validAndXfeMaxMsgSizeFromMBtoB(int inMaxMsgSizeInMB) {
+ return MixedUtils.mid(inMaxMsgSizeInMB,
+ TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB,
+ TBaseConstants.META_MAX_ALLOWED_MESSAGE_SIZE_MB)
+ * TBaseConstants.META_MB_UNIT_SIZE;
}
- public static int validAndGetMaxMsgSize(int inMaxMsgSize) {
- return mid(inMaxMsgSize,
- TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
- TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT);
+ public static int validAndGetMaxMsgSizeInB(int inMaxMsgSizeInB) {
+ return MixedUtils.mid(inMaxMsgSizeInB,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT);
}
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcConstants.java b/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcConstants.java
index e99ffc3..3a150b3 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcConstants.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corerpc/RpcConstants.java
@@ -18,6 +18,9 @@
package org.apache.tubemq.corerpc;
+import org.apache.tubemq.corebase.TBaseConstants;
+
+
public final class RpcConstants {
public static final String RPC_CODEC = "rpc.codec";
@@ -66,7 +69,8 @@ public final class RpcConstants {
public static final int RPC_PROTOCOL_BEGIN_TOKEN = 0xFF7FF4FE;
public static final int RPC_MAX_BUFFER_SIZE = 8192;
public static final int MAX_FRAME_MAX_LIST_SIZE =
- (int) ((1024 * 1024 * 8) / RPC_MAX_BUFFER_SIZE);
+ (int) ((TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT
+ + TBaseConstants.META_MB_UNIT_SIZE * 8) / RPC_MAX_BUFFER_SIZE);
public static final int RPC_FLAG_MSG_TYPE_REQUEST = 0x0;
public static final int RPC_FLAG_MSG_TYPE_RESPONSE = 0x1;
diff --git a/tubemq-core/src/main/proto/MasterService.proto b/tubemq-core/src/main/proto/MasterService.proto
index 27e5496..b97f3fc 100644
--- a/tubemq-core/src/main/proto/MasterService.proto
+++ b/tubemq-core/src/main/proto/MasterService.proto
@@ -66,7 +66,7 @@ message ApprovedClientConfig {
optional int32 maxMsgSize = 2;
}
-message ClusterDefConfig {
+message ClusterConfig {
required int64 configId = 1;
optional int32 maxMsgSize = 2;
}
@@ -222,7 +222,7 @@ message RegisterRequestB2M {
optional int32 qryPriorityId = 12;
optional int32 tlsPort = 13;
optional MasterCertificateInfo authInfo = 14;
- optional ClusterDefConfig clsDefConfig = 15;
+ optional ClusterConfig clsConfig = 15;
}
message RegisterResponseM2B {
@@ -245,7 +245,7 @@ message RegisterResponseM2B {
optional int32 qryPriorityId = 15;
optional MasterAuthorizedInfo authorizedInfo = 16; /* Deprecated */
optional MasterBrokerAuthorizedInfo brokerAuthorizedInfo = 17;
- optional ClusterDefConfig clsDefConfig = 18;
+ optional ClusterConfig clsConfig = 18;
}
message HeartRequestB2M {
@@ -266,7 +266,7 @@ message HeartRequestB2M {
optional int64 flowCheckId = 13;
optional int32 qryPriorityId = 14;
optional MasterCertificateInfo authInfo = 15;
- optional ClusterDefConfig clsDefConfig = 16;
+ optional ClusterConfig clsConfig = 16;
}
message HeartResponseM2B {
@@ -292,7 +292,7 @@ message HeartResponseM2B {
optional int32 qryPriorityId = 17;
optional MasterAuthorizedInfo authorizedInfo = 18; /* Deprecated */
optional MasterBrokerAuthorizedInfo brokerAuthorizedInfo = 19;
- optional ClusterDefConfig clsDefConfig = 20;
+ optional ClusterConfig clsConfig = 20;
}
message CloseRequestB2M {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
index f9b242f..60490c6 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
@@ -55,6 +55,7 @@ import org.apache.tubemq.corerpc.RpcConstants;
import org.apache.tubemq.corerpc.service.BrokerReadService;
import org.apache.tubemq.corerpc.service.BrokerWriteService;
import org.apache.tubemq.server.Server;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.metadata.MetadataManager;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
@@ -633,10 +634,10 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg("data length is zero!");
return builder.build();
}
- if (dataLength > TBaseConstants.META_MAX_MESSAGE_DATA_SIZE + 1024) {
+ if (dataLength > ClusterConfigHolder.getMaxMsgSize()) {
builder.setErrCode(TErrCodeConstants.BAD_REQUEST);
builder.setErrMsg(strBuffer.append("data length over max length, allowed max length is ")
- .append(TBaseConstants.META_MAX_MESSAGE_DATA_SIZE + 1024)
+ .append(ClusterConfigHolder.getMaxMsgSize())
.append(", data length is ").append(dataLength).toString());
return builder.build();
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/TubeBroker.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/TubeBroker.java
index 556cbba..690f0d7 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/TubeBroker.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/TubeBroker.java
@@ -46,6 +46,7 @@ import org.apache.tubemq.corerpc.service.MasterService;
import org.apache.tubemq.server.Stoppable;
import org.apache.tubemq.server.broker.exception.StartupException;
import org.apache.tubemq.server.broker.metadata.BrokerMetadataManager;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.metadata.MetadataManager;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.nodeinfo.ConsumerNodeInfo;
@@ -223,68 +224,8 @@ public class TubeBroker implements Stoppable {
}
isKeepAlive.set(true);
heartbeatErrors.set(0);
- FlowCtrlRuleHandler flowCtrlRuleHandler =
- metadataManager.getFlowCtrlRuleHandler();
- long flowCheckId = flowCtrlRuleHandler.getFlowCtrlId();
- int qryPriorityId = flowCtrlRuleHandler.getQryPriorityId();
- ServiceStatusHolder
- .setReadWriteServiceStatus(response.getStopRead(),
- response.getStopWrite(), "Master");
- if (response.hasFlowCheckId()) {
- qryPriorityId = response.hasQryPriorityId()
- ? response.getQryPriorityId() : qryPriorityId;
- if (response.getFlowCheckId() != flowCheckId) {
- flowCheckId = response.getFlowCheckId();
- try {
- flowCtrlRuleHandler
- .updateDefFlowCtrlInfo(qryPriorityId,
- flowCheckId, response.getFlowControlInfo());
- } catch (Exception e1) {
- logger.warn(
- "[HeartBeat response] found parse flowCtrl rules failure", e1);
- }
- }
- if (qryPriorityId != flowCtrlRuleHandler.getQryPriorityId()) {
- flowCtrlRuleHandler.setQryPriorityId(qryPriorityId);
- }
- }
- requireReportConf = response.getNeedReportData();
StringBuilder sBuilder = new StringBuilder(512);
- if (response.getTakeConfInfo()) {
- logger.info(sBuilder
- .append("[HeartBeat response] received broker metadata info: brokerConfId=")
- .append(response.getCurBrokerConfId())
- .append(",stopWrite=").append(response.getStopWrite())
- .append(",stopRead=").append(response.getStopRead())
- .append(",configCheckSumId=").append(response.getConfCheckSumId())
- .append(",hasFlowCtrl=").append(response.hasFlowCheckId())
- .append(",curFlowCtrlId=").append(flowCheckId)
- .append(",curQryPriorityId=").append(qryPriorityId)
- .append(",brokerDefaultConfInfo=")
- .append(response.getBrokerDefaultConfInfo())
- .append(",brokerTopicSetConfList=")
- .append(response.getBrokerTopicSetConfInfoList().toString()).toString());
- sBuilder.delete(0, sBuilder.length());
- metadataManager
- .updateBrokerTopicConfigMap(response.getCurBrokerConfId(),
- response.getConfCheckSumId(), response.getBrokerDefaultConfInfo(),
- response.getBrokerTopicSetConfInfoList(), false, sBuilder);
- }
- if (response.hasBrokerAuthorizedInfo()) {
- serverAuthHandler.appendVisitToken(response.getBrokerAuthorizedInfo());
- }
- boolean needProcess =
- metadataManager.updateBrokerRemoveTopicMap(
- response.getTakeRemoveTopicInfo(),
- response.getRemoveTopicConfInfoList(), sBuilder);
- if (needProcess) {
- new Thread() {
- @Override
- public void run() {
- storeManager.removeTopicStore();
- }
- }.start();
- }
+ procConfigFromHeartBeat(sBuilder, response);
} catch (Throwable t) {
isKeepAlive.set(false);
heartbeatErrors.incrementAndGet();
@@ -355,6 +296,92 @@ public class TubeBroker implements Stoppable {
.append(TubeServerVersion.BROKER_VERSION).toString();
}
+ private void procConfigFromHeartBeat(StringBuilder sBuilder,
+ HeartResponseM2B response) {
+ // process service status
+ ServiceStatusHolder
+ .setReadWriteServiceStatus(response.getStopRead(),
+ response.getStopWrite(), "Master");
+ // process flow controller rules
+ FlowCtrlRuleHandler flowCtrlRuleHandler =
+ metadataManager.getFlowCtrlRuleHandler();
+ long flowCheckId = flowCtrlRuleHandler.getFlowCtrlId();
+ int qryPriorityId = flowCtrlRuleHandler.getQryPriorityId();
+ if (response.hasFlowCheckId()) {
+ qryPriorityId = response.hasQryPriorityId()
+ ? response.getQryPriorityId() : qryPriorityId;
+ if (response.getFlowCheckId() != flowCheckId) {
+ flowCheckId = response.getFlowCheckId();
+ try {
+ flowCtrlRuleHandler
+ .updateDefFlowCtrlInfo(qryPriorityId,
+ flowCheckId, response.getFlowControlInfo());
+ } catch (Exception e1) {
+ logger.warn(
+ "[HeartBeat response] found parse flowCtrl rules failure", e1);
+ }
+ }
+ if (qryPriorityId != flowCtrlRuleHandler.getQryPriorityId()) {
+ flowCtrlRuleHandler.setQryPriorityId(qryPriorityId);
+ }
+ }
+ // update configure report requirement
+ requireReportConf = response.getNeedReportData();
+ // update cluster setting
+ if (response.hasClsConfig()) {
+ long configId = response.getClsConfig().getConfigId();
+ if (configId != ClusterConfigHolder.getConfigId()) {
+ ClusterConfigHolder.updClusterSetting(response.getClsConfig());
+ logger.info(sBuilder
+ .append("[HeartBeat response] received cluster configure changed,")
+ .append(",hasClsConfig=").append(response.hasClsConfig())
+ .append(",curClusterConfigId=").append(ClusterConfigHolder.getConfigId())
+ .append(",curMaxMsgSize=").append(ClusterConfigHolder.getMaxMsgSize())
+ .append(",minMemCacheSize=")
+ .append(ClusterConfigHolder.getMinMemCacheSize())
+ .toString());
+ sBuilder.delete(0, sBuilder.length());
+ }
+ }
+ if (response.getTakeConfInfo()) {
+ logger.info(sBuilder
+ .append("[HeartBeat response] received broker metadata info: brokerConfId=")
+ .append(response.getCurBrokerConfId())
+ .append(",stopWrite=").append(response.getStopWrite())
+ .append(",stopRead=").append(response.getStopRead())
+ .append(",configCheckSumId=").append(response.getConfCheckSumId())
+ .append(",hasFlowCtrl=").append(response.hasFlowCheckId())
+ .append(",curFlowCtrlId=").append(flowCheckId)
+ .append(",curQryPriorityId=").append(qryPriorityId)
+ .append(",brokerDefaultConfInfo=")
+ .append(response.getBrokerDefaultConfInfo())
+ .append(",brokerTopicSetConfList=")
+ .append(response.getBrokerTopicSetConfInfoList().toString()).toString());
+ sBuilder.delete(0, sBuilder.length());
+ metadataManager
+ .updateBrokerTopicConfigMap(response.getCurBrokerConfId(),
+ response.getConfCheckSumId(), response.getBrokerDefaultConfInfo(),
+ response.getBrokerTopicSetConfInfoList(), false, sBuilder);
+ }
+ // update auth info
+ if (response.hasBrokerAuthorizedInfo()) {
+ serverAuthHandler.appendVisitToken(response.getBrokerAuthorizedInfo());
+ }
+ // process topic deletion
+ boolean needProcess =
+ metadataManager.updateBrokerRemoveTopicMap(
+ response.getTakeRemoveTopicInfo(),
+ response.getRemoveTopicConfInfoList(), sBuilder);
+ if (needProcess) {
+ new Thread() {
+ @Override
+ public void run() {
+ storeManager.removeTopicStore();
+ }
+ }.start();
+ }
+ }
+
/***
* Register to master. Try multi times if failed.
*
@@ -374,53 +401,7 @@ public class TubeBroker implements Stoppable {
.append("Register to master failed! The error message is ")
.append(response.getErrMsg()).toString());
}
- ServiceStatusHolder
- .setReadWriteServiceStatus(response.getStopRead(),
- response.getStopWrite(), "Master");
- FlowCtrlRuleHandler flowCtrlRuleHandler =
- metadataManager.getFlowCtrlRuleHandler();
- if (response.hasFlowCheckId()) {
- int qryPriorityId = response.hasQryPriorityId()
- ? response.getQryPriorityId() : flowCtrlRuleHandler.getQryPriorityId();
- if (response.getFlowCheckId() != flowCtrlRuleHandler.getFlowCtrlId()) {
- try {
- flowCtrlRuleHandler
- .updateDefFlowCtrlInfo(response.getQryPriorityId(),
- response.getFlowCheckId(), response.getFlowControlInfo());
- } catch (Exception e1) {
- logger.warn("[Register response] found parse flowCtrl rules failure", e1);
- }
- }
- if (qryPriorityId != flowCtrlRuleHandler.getQryPriorityId()) {
- flowCtrlRuleHandler.setQryPriorityId(qryPriorityId);
- }
- }
- updateEnableBrokerFunInfo(response);
- logger.info(sBuilder
- .append("[Register response] received broker metadata info: brokerConfId=")
- .append(response.getCurBrokerConfId())
- .append(",stopWrite=").append(response.getStopWrite())
- .append(",stopRead=").append(response.getStopRead())
- .append(",configCheckSumId=").append(response.getConfCheckSumId())
- .append(",hasFlowCtrl=").append(response.hasFlowCheckId())
- .append(",enableVisitTokenCheck=")
- .append(serverAuthHandler.isEnableVisitTokenCheck())
- .append(",enableProduceAuthenticate=")
- .append(serverAuthHandler.isEnableProduceAuthenticate())
- .append(",enableProduceAuthorize=").append(serverAuthHandler.isEnableProduceAuthorize())
- .append(",enableConsumeAuthenticate=")
- .append(serverAuthHandler.isEnableConsumeAuthenticate())
- .append(",enableConsumeAuthorize=")
- .append(serverAuthHandler.isEnableConsumeAuthorize())
- .append(",curFlowCtrlId=").append(flowCtrlRuleHandler.getFlowCtrlId())
- .append(",curQryPriorityId=").append(flowCtrlRuleHandler.getQryPriorityId())
- .append(",brokerDefaultConfInfo=").append(response.getBrokerDefaultConfInfo())
- .append(",brokerTopicSetConfList=")
- .append(response.getBrokerTopicSetConfInfoList().toString()).toString());
- sBuilder.delete(0, sBuilder.length());
- metadataManager.updateBrokerTopicConfigMap(response.getCurBrokerConfId(),
- response.getConfCheckSumId(), response.getBrokerDefaultConfInfo(),
- response.getBrokerTopicSetConfInfoList(), true, sBuilder);
+ procConfigFromRegister(sBuilder, response);
isKeepAlive.set(true);
lastRegTime.set(System.currentTimeMillis());
break;
@@ -435,11 +416,80 @@ public class TubeBroker implements Stoppable {
}
}
- private void updateEnableBrokerFunInfo(final RegisterResponseM2B response) {
+
+ private void procConfigFromRegister(StringBuilder sBuilder,
+ final RegisterResponseM2B response) {
+ // process service status
+ ServiceStatusHolder
+ .setReadWriteServiceStatus(response.getStopRead(),
+ response.getStopWrite(), "Master");
+ // process flow controller rules
+ FlowCtrlRuleHandler flowCtrlRuleHandler =
+ metadataManager.getFlowCtrlRuleHandler();
+ if (response.hasFlowCheckId()) {
+ int qryPriorityId = response.hasQryPriorityId()
+ ? response.getQryPriorityId() : flowCtrlRuleHandler.getQryPriorityId();
+ if (response.getFlowCheckId() != flowCtrlRuleHandler.getFlowCtrlId()) {
+ try {
+ flowCtrlRuleHandler
+ .updateDefFlowCtrlInfo(response.getQryPriorityId(),
+ response.getFlowCheckId(), response.getFlowControlInfo());
+ } catch (Exception e1) {
+ logger.warn("[Register response] found parse flowCtrl rules failure", e1);
+ }
+ }
+ if (qryPriorityId != flowCtrlRuleHandler.getQryPriorityId()) {
+ flowCtrlRuleHandler.setQryPriorityId(qryPriorityId);
+ }
+ }
+ // update auth info
serverAuthHandler.configure(response.getEnableBrokerInfo());
if (response.hasBrokerAuthorizedInfo()) {
serverAuthHandler.appendVisitToken(response.getBrokerAuthorizedInfo());
}
+ // update cluster setting
+ if (response.hasClsConfig()) {
+ long configId = response.getClsConfig().getConfigId();
+ if (configId != ClusterConfigHolder.getConfigId()) {
+ ClusterConfigHolder.updClusterSetting(response.getClsConfig());
+ }
+ }
+ sBuilder.append("[Register response] received broker metadata info: brokerConfId=")
+ .append(response.getCurBrokerConfId())
+ .append(",stopWrite=").append(response.getStopWrite())
+ .append(",stopRead=").append(response.getStopRead())
+ .append(",configCheckSumId=").append(response.getConfCheckSumId())
+ .append(",hasFlowCtrl=").append(response.hasFlowCheckId())
+ .append(",curFlowCtrlId=").append(flowCtrlRuleHandler.getFlowCtrlId())
+ .append(",curQryPriorityId=").append(flowCtrlRuleHandler.getQryPriorityId())
+ .append(",hasClsConfig=").append(response.hasClsConfig())
+ .append(",curClusterConfigId=").append(ClusterConfigHolder.getConfigId())
+ .append(",curMaxMsgSize=").append(ClusterConfigHolder.getMaxMsgSize())
+ .append(",minMemCacheSize=").append(ClusterConfigHolder.getMinMemCacheSize())
+ .append(",enableVisitTokenCheck=")
+ .append(serverAuthHandler.isEnableVisitTokenCheck())
+ .append(",enableProduceAuthenticate=")
+ .append(serverAuthHandler.isEnableProduceAuthenticate())
+ .append(",enableProduceAuthorize=").append(serverAuthHandler.isEnableProduceAuthorize())
+ .append(",enableConsumeAuthenticate=")
+ .append(serverAuthHandler.isEnableConsumeAuthenticate())
+ .append(",enableConsumeAuthorize=")
+ .append(serverAuthHandler.isEnableConsumeAuthorize())
+ .append(",brokerDefaultConfInfo=").append(response.getBrokerDefaultConfInfo())
+ .append(",brokerTopicSetConfList=")
+ .append(response.getBrokerTopicSetConfInfoList().toString()).toString();
+ sBuilder.delete(0, sBuilder.length());
+ metadataManager.updateBrokerTopicConfigMap(response.getCurBrokerConfId(),
+ response.getConfCheckSumId(), response.getBrokerDefaultConfInfo(),
+ response.getBrokerTopicSetConfInfoList(), true, sBuilder);
+ }
+
+ // build cluster configure info
+ private ClientMaster.ClusterConfig.Builder buildClusterConfig() {
+ ClientMaster.ClusterConfig.Builder defSetting =
+ ClientMaster.ClusterConfig.newBuilder();
+ defSetting.setConfigId(ClusterConfigHolder.getConfigId());
+ return defSetting;
}
/***
@@ -474,6 +524,7 @@ public class TubeBroker implements Stoppable {
if (authInfoBuilder != null) {
builder.setAuthInfo(authInfoBuilder.build());
}
+ builder.setClsConfig(buildClusterConfig());
logger.info(new StringBuilder(512)
.append("[Register request] current broker report info: brokerConfId=")
.append(metadataManager.getBrokerMetadataConfId())
@@ -517,6 +568,7 @@ public class TubeBroker implements Stoppable {
if (authInfoBuilder != null) {
builder.setAuthInfo(authInfoBuilder.build());
}
+ builder.setClsConfig(buildClusterConfig());
if (metadataManager.isBrokerMetadataChanged() || requireReportConf) {
builder.setTakeConfInfo(true);
builder.setBrokerDefaultConfInfo(metadataManager.getBrokerDefMetaConfInfo());
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
new file mode 100644
index 0000000..c72427d
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.broker.metadata;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.protobuf.generated.ClientMaster;
+import org.apache.tubemq.corebase.utils.MixedUtils;
+import org.apache.tubemq.server.broker.utils.DataStoreUtils;
+
+
+public class ClusterConfigHolder {
+ private static AtomicLong configId =
+ new AtomicLong(TBaseConstants.META_VALUE_UNDEFINED);
+ private static AtomicInteger maxMsgSize =
+ new AtomicInteger(TBaseConstants.META_MAX_MESSAGE_DATA_SIZE
+ + TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE);
+ private static AtomicInteger minMemCacheSize =
+ new AtomicInteger(TBaseConstants.META_MIN_MEM_BUFFER_SIZE);
+ private static AtomicInteger maxMsgStoreLength =
+ new AtomicInteger(TBaseConstants.META_MAX_MESSAGE_DATA_SIZE
+ + TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE
+ + DataStoreUtils.STORE_DATA_HEADER_LEN);
+
+ public ClusterConfigHolder() {
+
+ }
+
+ // set master returned configure
+ public static void updClusterSetting(ClientMaster.ClusterConfig clusterConfig) {
+ if (clusterConfig == null) {
+ return;
+ }
+ if (configId.get() != clusterConfig.getConfigId()) {
+ configId.set(clusterConfig.getConfigId());
+ if (clusterConfig.hasMaxMsgSize()) {
+ int tmpMaxSize = MixedUtils.mid(clusterConfig.getMaxMsgSize(),
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT)
+ + TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE;
+ if (tmpMaxSize != maxMsgSize.get()) {
+ maxMsgSize.set(tmpMaxSize);
+ minMemCacheSize.set(tmpMaxSize +
+ (tmpMaxSize % 4 + 1) * TBaseConstants.META_MESSAGE_SIZE_ADJUST);
+ maxMsgStoreLength.set(tmpMaxSize + DataStoreUtils.STORE_DATA_HEADER_LEN);
+ }
+ }
+ }
+ }
+
+ public static long getConfigId() {
+ return configId.get();
+ }
+
+ public static int getMaxMsgSize() {
+ return maxMsgSize.get();
+ }
+
+ public static int getMinMemCacheSize() {
+ return minMemCacheSize.get();
+ }
+
+ public static int getMaxMsgStoreLength() {
+ return maxMsgStoreLength.get();
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
index e610a8b..8f650b4 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
@@ -37,6 +37,7 @@ import org.apache.tubemq.corebase.TErrCodeConstants;
import org.apache.tubemq.corebase.protobuf.generated.ClientBroker;
import org.apache.tubemq.corebase.utils.ThreadUtils;
import org.apache.tubemq.server.broker.BrokerConfig;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.metadata.TopicMetadata;
import org.apache.tubemq.server.broker.msgstore.disk.GetMessageResult;
import org.apache.tubemq.server.broker.msgstore.disk.MsgFileStatisInfo;
@@ -129,7 +130,7 @@ public class MessageStore implements Closeable {
this.unflushThreshold.set(topicMetadata.getUnflushThreshold());
this.unflushDataHold.set(topicMetadata.getUnflushDataHold());
this.writeCacheMaxCnt = topicMetadata.getMemCacheMsgCnt();
- this.writeCacheMaxSize = topicMetadata.getMemCacheMsgSize();
+ this.writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata.getMemCacheMsgSize());
this.writeCacheFlushIntvl = topicMetadata.getMemCacheFlushIntvl();
int tmpIndexReadCnt = tubeConfig.getIndexTransCount() * partitionNum;
memMaxIndexReadCnt.set(tmpIndexReadCnt <= 6000
@@ -421,7 +422,7 @@ public class MessageStore implements Closeable {
writeCacheMutex.readLock().lock();
try {
writeCacheMaxCnt = topicMetadata.getMemCacheMsgCnt();
- writeCacheMaxSize = topicMetadata.getMemCacheMsgSize();
+ writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata.getMemCacheMsgSize());
writeCacheFlushIntvl = topicMetadata.getMemCacheFlushIntvl();
} finally {
writeCacheMutex.readLock().unlock();
@@ -603,6 +604,17 @@ public class MessageStore implements Closeable {
}
}
+ private int validAndGetMemCacheSize(int memCacheSize) {
+ if (memCacheSize <= ClusterConfigHolder.getMinMemCacheSize()) {
+ logger.info(new StringBuilder(512)
+ .append("[Data Store] writeCacheMaxSize changed, from ")
+ .append(memCacheSize).append(" to ")
+ .append(ClusterConfigHolder.getMinMemCacheSize()).toString());
+ memCacheSize = ClusterConfigHolder.getMinMemCacheSize();
+ }
+ return memCacheSize;
+ }
+
/***
* Append message and trigger flush operation.
*
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java
index 949d149..6fda352 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java
@@ -25,6 +25,7 @@ import java.nio.channels.FileChannel;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.tubemq.corebase.utils.CheckSum;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -365,7 +366,7 @@ public class FileSegment implements Segment {
itemNext = validBytes + DataStoreUtils.STORE_DATA_HEADER_LEN + itemMsglen;
if ((itemMsgToken != DataStoreUtils.STORE_DATA_TOKER_BEGIN_VALUE)
|| (itemMsglen <= 0)
- || (itemMsglen > DataStoreUtils.MAX_MSG_DATA_STORE_SIZE)
+ || (itemMsglen > ClusterConfigHolder.getMaxMsgSize())
|| (itemNext > totalBytes)) {
next = -1;
break;
@@ -437,7 +438,7 @@ public class FileSegment implements Segment {
if ((itemMsgPartId < 0)
|| (itemMsgOffset < 0)
|| (itemMsglen <= 0)
- || (itemMsglen > DataStoreUtils.STORE_MAX_MESSAGE_STORE_LEN)
+ || (itemMsglen > ClusterConfigHolder.getMaxMsgStoreLength())
|| (itemNext > totalBytes)) {
next = -1;
break;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java
index 400b666..fe23dd3 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java
@@ -35,6 +35,7 @@ import org.apache.tubemq.corebase.TErrCodeConstants;
import org.apache.tubemq.corebase.protobuf.generated.ClientBroker;
import org.apache.tubemq.corebase.utils.ServiceStatusHolder;
import org.apache.tubemq.server.broker.BrokerConfig;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.stats.CountItem;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
@@ -273,7 +274,7 @@ public class MsgFileStore implements Closeable {
// skip when mismatch condition
if (curIndexDataOffset < 0
|| curIndexDataSize <= 0
- || curIndexDataSize > DataStoreUtils.STORE_MAX_MESSAGE_STORE_LEN
+ || curIndexDataSize > ClusterConfigHolder.getMaxMsgStoreLength()
|| curIndexDataOffset < curDataMinOffset) {
readedOffset = curIndexOffset + DataStoreUtils.STORE_INDEX_HEAD_LEN;
continue;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/mem/MsgMemStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/mem/MsgMemStore.java
index 6da3d27..27eef32 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/mem/MsgMemStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/mem/MsgMemStore.java
@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TErrCodeConstants;
import org.apache.tubemq.server.broker.BrokerConfig;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.msgstore.disk.MsgFileStore;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.apache.tubemq.server.common.utils.AppendResult;
@@ -224,7 +225,7 @@ public class MsgMemStore implements Closeable {
if ((cDataOffset < 0)
|| (cDataSize <= 0)
|| (cDataOffset >= currDataOffset)
- || (cDataSize > TBaseConstants.META_MAX_MESSAGE_DATA_SIZE + 1024)
+ || (cDataSize > ClusterConfigHolder.getMaxMsgSize())
|| (cDataOffset + cDataSize > currDataOffset)) {
readedSize += DataStoreUtils.STORE_INDEX_HEAD_LEN;
continue;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index 05688a7..2ed75c3 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -88,8 +88,8 @@ public enum WebFieldDef {
ADMINAUTHTOKEN(23, "confModAuthToken", "authToken", WebFieldType.STRING,
"Admin api operation authorization code",
TServerConstants.CFG_MODAUTHTOKEN_MAX_LENGTH),
- MAXMSGSIZE(24, "maxMsgSize", "maxMsgSize", WebFieldType.INT,
- "Max allowed message size", RegexDef.TMP_NUMBER),
+ MAXMSGSIZE(24, "maxMsgSizeInMB", "maxMsgSizeInMB", WebFieldType.INT,
+ "Max allowed message size, unit MB", RegexDef.TMP_NUMBER),
CREATEDATE(25, "createDate", "cDate", WebFieldType.STRING,
"Record creation date", TBaseConstants.META_MAX_DATEVALUE_LENGTH),
MODIFYDATE(26, "modifyDate", "mDate", WebFieldType.STRING,
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
index aa87940..ed2e122 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
@@ -32,6 +32,8 @@ import org.apache.tubemq.corebase.cluster.ConsumerInfo;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.broker.metadata.MetadataManager;
import org.apache.tubemq.server.broker.metadata.TopicMetadata;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
+import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.master.MasterConfig;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumeGroupSettingEntity;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerConfManager;
@@ -576,4 +578,35 @@ public class PBParameterUtils {
retResult.setCheckData(tmpValue);
return retResult;
}
+
+ /**
+ * Check the string parameter
+ *
+ * @param fieldDef the field to be checked
+ * @param paramValue the field value to be checked
+ * @param strBuffer the string pool construct the result
+ * @param result the checked result
+ * @return result success or failure
+ */
+ public static boolean getStringParameter(WebFieldDef fieldDef,
+ String paramValue,
+ StringBuilder strBuffer,
+ ProcessResult result) {
+ if (TStringUtils.isBlank(paramValue)) {
+ result.setFailResult(strBuffer.append("Request miss necessary ")
+ .append(fieldDef.name).append(" data!").toString());
+ strBuffer.delete(0, strBuffer.length());
+ return result.success;
+ }
+ String tmpValue = paramValue.trim();
+ if (tmpValue.length() > fieldDef.valMaxLen) {
+ result.setFailResult(strBuffer.append(fieldDef.name)
+ .append("'s length over max value, allowed max length is ")
+ .append(fieldDef.valMaxLen).toString());
+ strBuffer.delete(0, strBuffer.length());
+ return result.success;
+ }
+ result.setSuccResult(tmpValue);
+ return result.success;
+ }
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index 385fe22..d4b1e20 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -297,37 +297,11 @@ public class WebParameterUtils {
WebFieldDef fieldDef,
boolean required,
ProcessResult result) {
- if (!getStringParamValue(req, fieldDef,
- required, null, result)) {
- return result.success;
- }
- Set<Integer> tgtValueSet = new HashSet<Integer>();
- if (fieldDef.isCompFieldType()) {
- Set<String> valItemSet = (Set<String>) result.retData1;
- if (valItemSet.isEmpty()) {
- result.setSuccResult(tgtValueSet);
- return result.success;
- }
- for (String itemVal : valItemSet) {
- if (!checkIntValueNorms(fieldDef,
- itemVal, false, -1, result)) {
- return result.success;
- }
- tgtValueSet.add((Integer) result.retData1);
- }
- } else {
- String paramValue = (String) result.retData1;
- if (paramValue == null) {
- result.setSuccResult(tgtValueSet);
- return result.success;
- }
- if (!checkIntValueNorms(fieldDef,
- paramValue, false, -1, result)) {
- tgtValueSet.add((Integer) result.retData1);
- }
- }
- result.setSuccResult(tgtValueSet);
- return result.success;
+ return getIntParamValue(req, fieldDef, required,
+ false, TBaseConstants.META_VALUE_UNDEFINED,
+ false, TBaseConstants.META_VALUE_UNDEFINED,
+ false, TBaseConstants.META_VALUE_UNDEFINED,
+ result);
}
/**
@@ -342,11 +316,48 @@ public class WebParameterUtils {
* @return process result
*/
public static boolean getIntParamValue(HttpServletRequest req,
- WebFieldDef fieldDef,
- boolean required,
- int defValue,
- int minValue,
- ProcessResult result) {
+ WebFieldDef fieldDef,
+ boolean required,
+ int defValue,
+ int minValue,
+ ProcessResult result) {
+ return getIntParamValue(req, fieldDef, required, true, defValue,
+ true, minValue, false, TBaseConstants.META_VALUE_UNDEFINED, result);
+ }
+
+ /**
+ * Parse the parameter value from an object value to a integer value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if the field not exist
+ * @param minValue min value required
+ * @param minValue max value allowed
+ * @param result process result of parameter value
+ * @return process result
+ */
+ public static boolean getIntParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ int defValue,
+ int minValue,
+ int maxValue,
+ ProcessResult result) {
+ return getIntParamValue(req, fieldDef, required, true, defValue,
+ true, minValue, true, maxValue, result);
+ }
+
+ private static boolean getIntParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ boolean hasDefVal,
+ int defValue,
+ boolean hasMinVal,
+ int minValue,
+ boolean hasMaxVal,
+ int maxValue,
+ ProcessResult result) {
if (!getStringParamValue(req, fieldDef, required, null, result)) {
return result.success;
}
@@ -354,13 +365,15 @@ public class WebParameterUtils {
Set<Integer> tgtValueSet = new HashSet<Integer>();
Set<String> valItemSet = (Set<String>) result.retData1;
if (valItemSet.isEmpty()) {
- tgtValueSet.add(defValue);
+ if (hasDefVal) {
+ tgtValueSet.add(defValue);
+ }
result.setSuccResult(tgtValueSet);
return result.success;
}
for (String itemVal : valItemSet) {
- if (!checkIntValueNorms(fieldDef,
- itemVal, true, minValue, result)) {
+ if (!checkIntValueNorms(fieldDef, itemVal,
+ hasMinVal, minValue, hasMaxVal, maxValue, result)) {
return result.success;
}
tgtValueSet.add((Integer) result.retData1);
@@ -369,11 +382,13 @@ public class WebParameterUtils {
} else {
String paramValue = (String) result.retData1;
if (paramValue == null) {
- result.setSuccResult(defValue);
+ if (hasDefVal) {
+ result.setSuccResult(defValue);
+ }
return result.success;
}
- checkIntValueNorms(fieldDef,
- paramValue, true, minValue, result);
+ checkIntValueNorms(fieldDef, paramValue,
+ hasMinVal, minValue, hasMinVal, maxValue, result);
}
return result.success;
}
@@ -691,6 +706,8 @@ public class WebParameterUtils {
* @param paramValue the parameter value
* @param hasMinVal whether there is a minimum
* param minValue the parameter min value
+ * @param hasMaxVal whether there is a maximum
+ * param maxValue the parameter max value
* @param result process result
* @return check result for string value of parameter
*/
@@ -698,6 +715,8 @@ public class WebParameterUtils {
String paramValue,
boolean hasMinVal,
int minValue,
+ boolean hasMaxVal,
+ int maxValue,
ProcessResult result) {
try {
int paramIntVal = Integer.parseInt(paramValue);
@@ -707,6 +726,12 @@ public class WebParameterUtils {
.append(" value must >= ").append(minValue).toString());
return false;
}
+ if (hasMaxVal && paramIntVal > maxValue) {
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" value must <= ").append(maxValue).toString());
+ return false;
+ }
result.setSuccResult(paramIntVal);
} catch (Throwable e) {
result.setFailResult(new StringBuilder(512)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
index 4299bc7..e7327cb 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
@@ -48,6 +48,7 @@ import org.apache.tubemq.corebase.cluster.ProducerInfo;
import org.apache.tubemq.corebase.cluster.SubscribeInfo;
import org.apache.tubemq.corebase.cluster.TopicInfo;
import org.apache.tubemq.corebase.config.TLSConfig;
+import org.apache.tubemq.corebase.protobuf.generated.ClientMaster;
import org.apache.tubemq.corebase.protobuf.generated.ClientMaster.CloseRequestB2M;
import org.apache.tubemq.corebase.protobuf.generated.ClientMaster.CloseRequestC2M;
import org.apache.tubemq.corebase.protobuf.generated.ClientMaster.CloseRequestP2M;
@@ -101,6 +102,7 @@ import org.apache.tubemq.server.master.balance.DefaultLoadBalancer;
import org.apache.tubemq.server.master.balance.LoadBalancer;
import org.apache.tubemq.server.master.bdbstore.DefaultBdbStoreService;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbClusterSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbGroupFlowCtrlEntity;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerConfManager;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerInfoHolder;
@@ -348,6 +350,11 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
builder.setBrokerCheckSum(this.defaultBrokerConfManager.getBrokerInfoCheckSum());
builder.addAllBrokerInfos(this.defaultBrokerConfManager.getBrokersMap(overtls).values());
builder.setAuthorizedInfo(genAuthorizedInfo(certResult.authorizedToken, false).build());
+ ClientMaster.ApprovedClientConfig.Builder clientConfigBuilder =
+ buildApprovedClientConfig(request.getAppdConfig());
+ if (clientConfigBuilder != null) {
+ builder.setAppdConfig(clientConfigBuilder);
+ }
logger.info(strBuffer.append("[Producer Register] ").append(producerId)
.append(", isOverTLS=").append(overtls)
.append(", clientJDKVer=").append(clientJdkVer).toString());
@@ -436,6 +443,11 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if (defaultBrokerConfManager.getBrokerInfoCheckSum() != inBrokerCheckSum) {
builder.addAllBrokerInfos(defaultBrokerConfManager.getBrokersMap(overtls).values());
}
+ ClientMaster.ApprovedClientConfig.Builder clientConfigBuilder =
+ buildApprovedClientConfig(request.getAppdConfig());
+ if (clientConfigBuilder != null) {
+ builder.setAppdConfig(clientConfigBuilder);
+ }
if (logger.isDebugEnabled()) {
logger.debug(strBuffer.append("[Push Producer's available topic count:]")
.append(producerId).append(TokenConstants.LOG_SEG_SEP)
@@ -1071,6 +1083,11 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
builder.setBrokerDefaultConfInfo(brokerStatusInfo.getLastPushBrokerDefaultConfInfo());
builder.addAllBrokerTopicSetConfInfo(brokerStatusInfo.getLastPushBrokerTopicSetConfInfo());
builder.setSsdStoreId(TBaseConstants.META_VALUE_UNDEFINED);
+ ClientMaster.ClusterConfig.Builder clusterConfigBuilder =
+ buildClusterConfig(request.getClsConfig());
+ if (clusterConfigBuilder != null) {
+ builder.setClsConfig(clusterConfigBuilder);
+ }
if (request.hasFlowCheckId()) {
BdbGroupFlowCtrlEntity bdbGroupFlowCtrlEntity =
defaultBrokerConfManager.getBdbDefFlowCtrl();
@@ -1259,6 +1276,11 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
}
}
brokerHolder.setBrokerHeartBeatReqStatus(brokerInfo.getBrokerId(), builder);
+ ClientMaster.ClusterConfig.Builder clusterConfigBuilder =
+ buildClusterConfig(request.getClsConfig());
+ if (clusterConfigBuilder != null) {
+ builder.setClsConfig(clusterConfigBuilder);
+ }
builder.setTakeRemoveTopicInfo(true);
builder.addAllRemoveTopicConfInfo(defaultBrokerConfManager
.getBrokerRemovedTopicStrConfigInfo(bdbBrokerConfEntity));
@@ -2295,6 +2317,56 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
}
/**
+ * build approved client configure
+ *
+ * @param inClientConfig client reported Configure info
+ * @return ApprovedClientConfig
+ */
+ private ClientMaster.ApprovedClientConfig.Builder buildApprovedClientConfig(
+ ClientMaster.ApprovedClientConfig inClientConfig) {
+ ClientMaster.ApprovedClientConfig.Builder outClientConfig = null;
+ if (inClientConfig != null) {
+ outClientConfig = ClientMaster.ApprovedClientConfig.newBuilder();
+ BdbClusterSettingEntity settingEntity =
+ this.defaultBrokerConfManager.getBdbClusterSetting();
+ if (settingEntity == null) {
+ outClientConfig.setConfigId(TBaseConstants.META_VALUE_UNDEFINED);
+ } else {
+ outClientConfig.setConfigId(settingEntity.getConfigId());
+ if (settingEntity.getConfigId() != inClientConfig.getConfigId()) {
+ outClientConfig.setMaxMsgSize(settingEntity.getMaxMsgSizeInB());
+ }
+ }
+ }
+ return outClientConfig;
+ }
+
+
+ /**
+ * build cluster configure info
+ *
+ * @param inClusterConfig broker reported Configure info
+ * @return ClusterConfig
+ */
+ private ClientMaster.ClusterConfig.Builder buildClusterConfig(
+ ClientMaster.ClusterConfig inClusterConfig) {
+ ClientMaster.ClusterConfig.Builder outClsConfig = null;
+ if (inClusterConfig != null) {
+ outClsConfig = ClientMaster.ClusterConfig.newBuilder();
+ BdbClusterSettingEntity settingEntity =
+ this.defaultBrokerConfManager.getBdbClusterSetting();
+ if (settingEntity == null) {
+ outClsConfig.setConfigId(TBaseConstants.META_VALUE_UNDEFINED);
+ } else {
+ outClsConfig.setConfigId(settingEntity.getConfigId());
+ if (settingEntity.getConfigId() != inClusterConfig.getConfigId()) {
+ outClsConfig.setMaxMsgSize(settingEntity.getMaxMsgSizeInB());
+ }
+ }
+ }
+ return outClsConfig;
+ }
+ /**
* Start balance chore
*
* @param master
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
index 588fd87..7b9b570 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
@@ -37,6 +37,7 @@ public class BdbClusterSettingEntity implements Serializable {
@PrimaryKey
private String recordKey = "";
+ private long configId = TBaseConstants.META_VALUE_UNDEFINED;
//broker tcp port
private int brokerPort = TBaseConstants.META_VALUE_UNDEFINED;
//broker tls port
@@ -48,20 +49,22 @@ public class BdbClusterSettingEntity implements Serializable {
//partition num
private int numPartitions = TBaseConstants.META_VALUE_UNDEFINED;
//flush disk threshold
- private int unflushDskThreshold = TBaseConstants.META_VALUE_UNDEFINED;
+ private int unflushThreshold = TBaseConstants.META_VALUE_UNDEFINED;
//flush disk interval
- private int unflushDksInterval = TBaseConstants.META_VALUE_UNDEFINED;
- //flush memory cache threshold
- private int unflushMemThreshold = TBaseConstants.META_VALUE_UNDEFINED;
- //flush memory cache interval
- private int unflushMemInterval = TBaseConstants.META_VALUE_UNDEFINED;
+ private int unflushInterval = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush disk data count
+ private int unflushDataHold = TBaseConstants.META_VALUE_UNDEFINED;
//flush memory cache count
- private int unflushMemCnt = TBaseConstants.META_VALUE_UNDEFINED;
+ private int memCacheMsgCntInK = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush memory cache interval
+ private int memCacheFlushIntvl = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush memory cache size
+ private int memCacheMsgSizeInMB = TBaseConstants.META_VALUE_UNDEFINED;
private boolean acceptPublish = true; //enable publish
private boolean acceptSubscribe = true; //enable subscribe
- private String deleteWhen = ""; //delete policy execute time
+ private String deletePolicy = ""; //delete policy execute time
private int qryPriorityId = TBaseConstants.META_VALUE_UNDEFINED;
- private int maxMsgSize = TBaseConstants.META_VALUE_UNDEFINED;
+ private int maxMsgSizeInB = TBaseConstants.META_VALUE_UNDEFINED;
private String attributes = ""; //extra attribute
private String modifyUser; //modify user
private Date modifyDate; //modify date
@@ -70,30 +73,34 @@ public class BdbClusterSettingEntity implements Serializable {
}
//Constructor
- public BdbClusterSettingEntity(String recordKey, int brokerPort, int brokerTLSPort,
- int brokerWebPort, int numTopicStores, int numPartitions,
- int unflushDskThreshold, int unflushDksInterval,
- int unflushMemThreshold, int unflushMemInterval,
- int unflushMemCnt, boolean acceptPublish,
- boolean acceptSubscribe, String deleteWhen,
- int qryPriorityId, int maxMsgSize, String attributes,
+ public BdbClusterSettingEntity(String recordKey, long configId, int brokerPort,
+ int brokerTLSPort, int brokerWebPort,
+ int numTopicStores, int numPartitions,
+ int unflushThreshold, int unflushInterval,
+ int unflushDataHold, int memCacheMsgCntInK,
+ int memCacheFlushIntvl, int memCacheMsgSizeInMB,
+ boolean acceptPublish, boolean acceptSubscribe,
+ String deletePolicy, int qryPriorityId,
+ int maxMsgSizeInB, String attributes,
String modifyUser, Date modifyDate) {
this.recordKey = recordKey;
+ this.configId = configId;
this.brokerPort = brokerPort;
this.brokerTLSPort = brokerTLSPort;
this.brokerWebPort = brokerWebPort;
this.numTopicStores = numTopicStores;
this.numPartitions = numPartitions;
- this.unflushDskThreshold = unflushDskThreshold;
- this.unflushDksInterval = unflushDksInterval;
- this.unflushMemThreshold = unflushMemThreshold;
- this.unflushMemInterval = unflushMemInterval;
- this.unflushMemCnt = unflushMemCnt;
+ this.unflushThreshold = unflushThreshold;
+ this.unflushInterval = unflushInterval;
+ this.unflushDataHold = unflushDataHold;
+ this.memCacheMsgCntInK = memCacheMsgCntInK;
+ this.memCacheFlushIntvl = memCacheFlushIntvl;
+ this.memCacheMsgSizeInMB = memCacheMsgSizeInMB;
this.acceptPublish = acceptPublish;
this.acceptSubscribe = acceptSubscribe;
- this.deleteWhen = deleteWhen;
+ this.deletePolicy = deletePolicy;
this.qryPriorityId = qryPriorityId;
- this.maxMsgSize = maxMsgSize;
+ this.maxMsgSizeInB = maxMsgSizeInB;
this.attributes = attributes;
this.modifyUser = modifyUser;
this.modifyDate = modifyDate;
@@ -107,6 +114,10 @@ public class BdbClusterSettingEntity implements Serializable {
return recordKey;
}
+ public long getConfigId() {
+ return configId;
+ }
+
public int getBrokerPort() {
return brokerPort;
}
@@ -147,44 +158,52 @@ public class BdbClusterSettingEntity implements Serializable {
this.numPartitions = numPartitions;
}
- public int getUnflushDskThreshold() {
- return unflushDskThreshold;
+ public int getUnflushThreshold() {
+ return unflushThreshold;
+ }
+
+ public void setUnflushThreshold(int unflushThreshold) {
+ this.unflushThreshold = unflushThreshold;
+ }
+
+ public int getUnflushInterval() {
+ return unflushInterval;
}
- public void setUnflushDskThreshold(int unflushDskThreshold) {
- this.unflushDskThreshold = unflushDskThreshold;
+ public void setUnflushInterval(int unflushInterval) {
+ this.unflushInterval = unflushInterval;
}
- public int getUnflushDksInterval() {
- return unflushDksInterval;
+ public int getUnflushDataHold() {
+ return unflushDataHold;
}
- public void setUnflushDksInterval(int unflushDksInterval) {
- this.unflushDksInterval = unflushDksInterval;
+ public void setUnflushDataHold(int unflushDataHold) {
+ this.unflushDataHold = unflushDataHold;
}
- public int getUnflushMemThreshold() {
- return unflushMemThreshold;
+ public int getMemCacheMsgCntInK() {
+ return memCacheMsgCntInK;
}
- public void setUnflushMemThreshold(int unflushMemThreshold) {
- this.unflushMemThreshold = unflushMemThreshold;
+ public void setMemCacheMsgCntInK(int memCacheMsgCntInK) {
+ this.memCacheMsgCntInK = memCacheMsgCntInK;
}
- public int getUnflushMemInterval() {
- return unflushMemInterval;
+ public int getMemCacheFlushIntvl() {
+ return memCacheFlushIntvl;
}
- public void setUnflushMemInterval(int unflushMemInterval) {
- this.unflushMemInterval = unflushMemInterval;
+ public void setMemCacheFlushIntvl(int memCacheFlushIntvl) {
+ this.memCacheFlushIntvl = memCacheFlushIntvl;
}
- public int getUnflushMemCnt() {
- return unflushMemCnt;
+ public int getMemCacheMsgSizeInMB() {
+ return memCacheMsgSizeInMB;
}
- public void setUnflushMemCnt(int unflushMemCnt) {
- this.unflushMemCnt = unflushMemCnt;
+ public void setMemCacheMsgSizeInMB(int memCacheMsgSizeInMB) {
+ this.memCacheMsgSizeInMB = memCacheMsgSizeInMB;
}
public boolean isAcceptPublish() {
@@ -203,12 +222,12 @@ public class BdbClusterSettingEntity implements Serializable {
this.acceptSubscribe = acceptSubscribe;
}
- public String getDeleteWhen() {
- return deleteWhen;
+ public String getDeletePolicy() {
+ return deletePolicy;
}
- public void setDeleteWhen(String deleteWhen) {
- this.deleteWhen = deleteWhen;
+ public void setDeletePolicy(String deletePolicy) {
+ this.deletePolicy = deletePolicy;
}
public int getQryPriorityId() {
@@ -219,12 +238,12 @@ public class BdbClusterSettingEntity implements Serializable {
this.qryPriorityId = qryPriorityId;
}
- public int getMaxMsgSize() {
- return maxMsgSize;
+ public int getMaxMsgSizeInB() {
+ return maxMsgSizeInB;
}
- public void setMaxMsgSize(int maxMsgSize) {
- this.maxMsgSize = maxMsgSize;
+ public void setMaxMsgSizeInB(int maxMsgSizeInB) {
+ this.maxMsgSizeInB = maxMsgSizeInB;
}
public String getAttributes() {
@@ -236,6 +255,7 @@ public class BdbClusterSettingEntity implements Serializable {
}
public void setModifyInfo(String modifyUser, Date modifyDate) {
+ this.configId = System.currentTimeMillis();
this.modifyUser = modifyUser;
this.modifyDate = modifyDate;
}
@@ -255,23 +275,30 @@ public class BdbClusterSettingEntity implements Serializable {
* @return
*/
public StringBuilder toJsonString(final StringBuilder sBuilder) {
- return sBuilder.append("{\"type\":\"BdbClusterSettingEntity\",")
+ sBuilder.append("{\"type\":\"BdbClusterSettingEntity\",")
.append("\"recordKey\":\"").append(recordKey).append("\"")
+ .append(",\"configId\":").append(configId)
.append(",\"brokerPort\":").append(brokerPort)
.append(",\"brokerTLSPort\":").append(brokerTLSPort)
.append(",\"brokerWebPort\":").append(brokerWebPort)
.append(",\"numTopicStores\":").append(numTopicStores)
.append(",\"numPartitions\":").append(numPartitions)
- .append(",\"unflushDskThreshold\":").append(unflushDskThreshold)
- .append(",\"unflushDksInterval\":").append(unflushDksInterval)
- .append(",\"unflushMemThreshold\":").append(unflushMemThreshold)
- .append(",\"unflushMemInterval\":").append(unflushMemInterval)
- .append(",\"unflushMemCnt\":").append(unflushMemCnt)
+ .append(",\"unflushThreshold\":").append(unflushThreshold)
+ .append(",\"unflushInterval\":").append(unflushInterval)
+ .append(",\"unflushDataHold\":").append(unflushDataHold)
+ .append(",\"memCacheMsgCntInK\":").append(memCacheMsgCntInK)
+ .append(",\"memCacheFlushIntvl\":").append(memCacheFlushIntvl)
+ .append(",\"memCacheMsgSizeInMB\":").append(memCacheMsgSizeInMB)
.append(",\"acceptPublish\":").append(acceptPublish)
.append(",\"acceptSubscribe\":").append(acceptSubscribe)
- .append(",\"deleteWhen\":\"").append(deleteWhen).append("\"")
- .append(",\"maxMsgSize\":").append(maxMsgSize)
- .append(",\"qryPriorityId\":").append(qryPriorityId)
+ .append(",\"deletePolicy\":\"").append(deletePolicy).append("\"")
+ .append(",\"maxMsgSizeInMB\":");
+ if (maxMsgSizeInB == TBaseConstants.META_VALUE_UNDEFINED) {
+ sBuilder.append(maxMsgSizeInB);
+ } else {
+ sBuilder.append(maxMsgSizeInB / TBaseConstants.META_MB_UNIT_SIZE);
+ }
+ return sBuilder.append(",\"qryPriorityId\":").append(qryPriorityId)
.append(",\"attributes\":\"").append(attributes).append("\"")
.append(",\"modifyUser\":\"").append(modifyUser).append("\"")
.append(",\"modifyDate\":\"")
@@ -281,23 +308,30 @@ public class BdbClusterSettingEntity implements Serializable {
@Override
public String toString() {
- return new ToStringBuilder(this)
+ ToStringBuilder sBuilder = new ToStringBuilder(this)
.append("recordKey", recordKey)
+ .append("configId", configId)
.append("brokerPort", brokerPort)
.append("brokerTLSPort", brokerTLSPort)
.append("brokerWebPort", brokerWebPort)
.append("numTopicStores", numTopicStores)
.append("numPartitions", numPartitions)
- .append("unflushDskThreshold", unflushDskThreshold)
- .append("unflushDksInterval", unflushDksInterval)
- .append("unflushMemThreshold", unflushMemThreshold)
- .append("unflushMemInterval", unflushMemInterval)
- .append("unflushMemCnt", unflushMemCnt)
+ .append("unflushThreshold", unflushThreshold)
+ .append("unflushInterval", unflushInterval)
+ .append("unflushDataHold", unflushDataHold)
+ .append("memCacheMsgCntInK", memCacheMsgCntInK)
+ .append("memCacheFlushIntvl", memCacheFlushIntvl)
+ .append("memCacheMsgSizeInMB", memCacheMsgSizeInMB)
.append("acceptPublish", acceptPublish)
.append("acceptSubscribe", acceptSubscribe)
- .append("deleteWhen", deleteWhen)
- .append("maxMsgSize", maxMsgSize)
- .append("qryPriorityId", qryPriorityId)
+ .append("deletePolicy", deletePolicy);
+ if (maxMsgSizeInB == TBaseConstants.META_VALUE_UNDEFINED) {
+ sBuilder.append("maxMsgSizeInMB", maxMsgSizeInB);
+ } else {
+ sBuilder.append("maxMsgSizeInMB",
+ maxMsgSizeInB / TBaseConstants.META_MB_UNIT_SIZE);
+ }
+ return sBuilder.append("qryPriorityId", qryPriorityId)
.append("attributes", attributes)
.append("modifyUser", modifyUser)
.append("modifyDate", modifyDate)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
index 9afe1aa..f2fece8 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
@@ -169,13 +169,14 @@ public class WebMasterInfoHandler extends AbstractWebHandler {
if (!WebParameterUtils.getIntParamValue(req,
WebFieldDef.MAXMSGSIZE, false,
TBaseConstants.META_VALUE_UNDEFINED,
- TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
+ TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB,
+ TBaseConstants.META_MAX_ALLOWED_MESSAGE_SIZE_MB,
result)) {
WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
return sBuilder;
}
- int maxMsgSize = (int) result.retData1;
- if (maxMsgSize != TBaseConstants.META_VALUE_UNDEFINED) {
+ int maxMsgSizeInMB = (int) result.retData1;
+ if (maxMsgSizeInMB != TBaseConstants.META_VALUE_UNDEFINED) {
dataChanged = true;
}
// check and get modify date
@@ -196,9 +197,9 @@ public class WebMasterInfoHandler extends AbstractWebHandler {
defClusterSetting = new BdbClusterSettingEntity();
}
defClusterSetting.setModifyInfo(modifyUser, modifyDate);
- if (maxMsgSize != TBaseConstants.META_VALUE_UNDEFINED) {
- defClusterSetting.setMaxMsgSize(
- SettingValidUtils.validAndGetMaxMsgSize(maxMsgSize));
+ if (maxMsgSizeInMB != TBaseConstants.META_VALUE_UNDEFINED) {
+ defClusterSetting.setMaxMsgSizeInB(
+ SettingValidUtils.validAndXfeMaxMsgSizeFromMBtoB(maxMsgSizeInMB));
}
try {
brokerConfManager.confSetBdbClusterDefSetting(defClusterSetting);
[incubator-tubemq] 46/49: [TUBEMQ-544]Adjust the LICENSE statement
in the client.conf files of Python and C/C++ SDK
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit f1bcd20a2093aab0c37cfbb3d9484bfefb345090
Author: gosonzhang <go...@tencent.com>
AuthorDate: Thu Jan 28 14:15:50 2021 +0800
[TUBEMQ-544]Adjust the LICENSE statement in the client.conf files of Python and C/C++ SDK
---
.../tubemq-client-cpp/conf/client.conf | 22 +++++++++++-----------
.../src/python/tubemq/client.conf | 22 +++++++++++-----------
2 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/tubemq-client-twins/tubemq-client-cpp/conf/client.conf b/tubemq-client-twins/tubemq-client-cpp/conf/client.conf
index 018dd5e..4591d0a 100644
--- a/tubemq-client-twins/tubemq-client-cpp/conf/client.conf
+++ b/tubemq-client-twins/tubemq-client-cpp/conf/client.conf
@@ -1,18 +1,18 @@
;
-; Tencent is pleased to support the open source community by making TubeMQ available.
+; Licensed to the Apache Software Foundation (ASF) under one or more
+; contributor license agreements. See the NOTICE file distributed with
+; this work for additional information regarding copyright ownership.
+; The ASF licenses this file to You under the Apache License, Version 2.0
+; (the "License"); you may not use this file except in compliance with
+; the License. You may obtain a copy of the License at
;
-; Copyright (C) 2012-2019 Tencent. All Rights Reserved.
-;
-; Licensed under the Apache License, Version 2.0 (the "License"); you may not use
-; this file except in compliance with the License. You may obtain a copy of the
-; License at
-;
-; https://opensource.org/licenses/Apache-2.0
+; http://www.apache.org/licenses/LICENSE-2.0
;
; Unless required by applicable law or agreed to in writing, software
-; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-; WARRANTIES OF ANY KIND, either express or implied. See the License for the
-; specific language governing permissions and limitations under the License.
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
;
[TubeMQ]
diff --git a/tubemq-client-twins/tubemq-client-python/src/python/tubemq/client.conf b/tubemq-client-twins/tubemq-client-python/src/python/tubemq/client.conf
index 237927d..e3f38ba 100644
--- a/tubemq-client-twins/tubemq-client-python/src/python/tubemq/client.conf
+++ b/tubemq-client-twins/tubemq-client-python/src/python/tubemq/client.conf
@@ -1,18 +1,18 @@
;
-; Tencent is pleased to support the open source community by making TubeMQ available.
+; Licensed to the Apache Software Foundation (ASF) under one or more
+; contributor license agreements. See the NOTICE file distributed with
+; this work for additional information regarding copyright ownership.
+; The ASF licenses this file to You under the Apache License, Version 2.0
+; (the "License"); you may not use this file except in compliance with
+; the License. You may obtain a copy of the License at
;
-; Copyright (C) 2012-2019 Tencent. All Rights Reserved.
-;
-; Licensed under the Apache License, Version 2.0 (the "License"); you may not use
-; this file except in compliance with the License. You may obtain a copy of the
-; License at
-;
-; https://opensource.org/licenses/Apache-2.0
+; http://www.apache.org/licenses/LICENSE-2.0
;
; Unless required by applicable law or agreed to in writing, software
-; distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-; WARRANTIES OF ANY KIND, either express or implied. See the License for the
-; specific language governing permissions and limitations under the License.
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
;
[TubeMQ]
[incubator-tubemq] 37/49: rm -Werror
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit eaf0f10ac68b839652b80a0b49dab57f78212c24
Author: jianxzhang <ji...@tencent.com>
AuthorDate: Mon Jan 11 19:16:34 2021 +0800
rm -Werror
---
tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt b/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt
index 94ac9f1..83d0ba5 100644
--- a/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt
+++ b/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt
@@ -24,7 +24,7 @@ project (TubeMQ)
find_package(Protobuf REQUIRED)
-SET(CMAKE_CXX_FLAGS "-std=c++11 -O2 -g -Wall -Werror -Wsign-compare -Wfloat-equal -fno-strict-aliasing -fPIC -DASIO_STANDALONE")
+SET(CMAKE_CXX_FLAGS "-std=c++11 -O2 -g -Wall -Wsign-compare -Wfloat-equal -fno-strict-aliasing -fPIC -DASIO_STANDALONE")
INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/proto)
[incubator-tubemq] 42/49: [TUBEMQ-515]Add cluster Topic view web api
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 4b138de7fef71c68b9fc35f7996890cb127c2129
Author: gosonzhang <go...@tencent.com>
AuthorDate: Sat Jan 16 17:47:22 2021 +0800
[TUBEMQ-515]Add cluster Topic view web api
---
resources/assets/scripts/topicList.js | 8 +-
.../server/common/utils/WebParameterUtils.java | 19 ++++
.../web/handler/WebBrokerTopicConfHandler.java | 33 ++----
.../master/web/handler/WebMasterInfoHandler.java | 115 ++++++++++++++++++++-
4 files changed, 146 insertions(+), 29 deletions(-)
diff --git a/resources/assets/scripts/topicList.js b/resources/assets/scripts/topicList.js
index b755a94..7dc000a 100644
--- a/resources/assets/scripts/topicList.js
+++ b/resources/assets/scripts/topicList.js
@@ -102,7 +102,7 @@
'false': '否',
'-': '-'
};
- var url = G_CONFIG.HOST + "?type=op_query&method=admin_query_topic_info&" + $.param(
+ var url = G_CONFIG.HOST + "?type=op_query&method=admin_query_cluster_topic_view&" + $.param(
opts);
if (!this.$topicListDataTable) {
@@ -126,7 +126,7 @@
return html;
}
}, {
- "data": "infoCount"
+ "data": "totalCfgBrokerCnt"
}, {
"data": "totalCfgNumPart"
}, {
@@ -166,13 +166,13 @@
+ '"><input type="checkbox" checked></span>';
}
}, {
- "data": "authData",
+ "data": "enableAuthControl",
"orderable": false,
"render": function (data,
type,
full,
meta) {
- var checked = data.enableAuthControl
+ var checked = data
=== true
? ' checked'
: '';
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index 27e61e1..161966d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -34,6 +34,7 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.TStatusConstants;
@@ -1401,6 +1402,24 @@ public class WebParameterUtils {
return strManageStatus;
}
+ public static Tuple2<Boolean, Boolean> getPubSubStatusByManageStatus(int manageStatus) {
+ boolean isAcceptPublish = false;
+ boolean isAcceptSubscribe = false;
+ if (manageStatus >= TStatusConstants.STATUS_MANAGE_ONLINE) {
+ if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE) {
+ isAcceptPublish = true;
+ isAcceptSubscribe = true;
+ } else if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE_NOT_WRITE) {
+ isAcceptPublish = false;
+ isAcceptSubscribe = true;
+ } else if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE_NOT_READ) {
+ isAcceptPublish = true;
+ isAcceptSubscribe = false;
+ }
+ }
+ return new Tuple2<>(isAcceptPublish, isAcceptSubscribe);
+ }
+
public static String date2yyyyMMddHHmmss(Date date) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
return sdf.format(date);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
index 9a4cf04..74a4ed0 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
@@ -33,6 +33,7 @@ import org.apache.tubemq.corebase.cluster.BrokerInfo;
import org.apache.tubemq.corebase.cluster.TopicInfo;
import org.apache.tubemq.corebase.utils.SettingValidUtils;
import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.TStatusConstants;
import org.apache.tubemq.server.common.fielddef.WebFieldDef;
@@ -628,18 +629,10 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
if (brokerConfEntity != null) {
int manageStatus = brokerConfEntity.getManageStatus();
strManageStatus = WebParameterUtils.getBrokerManageStatusStr(manageStatus);
- if (manageStatus >= TStatusConstants.STATUS_MANAGE_ONLINE) {
- if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE) {
- isAcceptPublish = true;
- isAcceptSubscribe = true;
- } else if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE_NOT_WRITE) {
- isAcceptPublish = false;
- isAcceptSubscribe = true;
- } else if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE_NOT_READ) {
- isAcceptPublish = true;
- isAcceptSubscribe = false;
- }
- }
+ Tuple2<Boolean, Boolean> pubSubStatus =
+ WebParameterUtils.getPubSubStatusByManageStatus(manageStatus);
+ isAcceptPublish = pubSubStatus.getF0();
+ isAcceptSubscribe = pubSubStatus.getF1();
}
BrokerInfo broker =
new BrokerInfo(entity.getBrokerId(), entity.getBrokerIp(), entity.getBrokerPort());
@@ -1243,18 +1236,10 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
if (brokerConfEntity != null) {
int manageStatus = brokerConfEntity.getManageStatus();
strManageStatus = WebParameterUtils.getBrokerManageStatusStr(manageStatus);
- if (manageStatus >= TStatusConstants.STATUS_MANAGE_ONLINE) {
- if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE) {
- isAcceptPublish = true;
- isAcceptSubscribe = true;
- } else if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE_NOT_WRITE) {
- isAcceptPublish = false;
- isAcceptSubscribe = true;
- } else if (manageStatus == TStatusConstants.STATUS_MANAGE_ONLINE_NOT_READ) {
- isAcceptPublish = true;
- isAcceptSubscribe = false;
- }
- }
+ Tuple2<Boolean, Boolean> pubSubStatus =
+ WebParameterUtils.getPubSubStatusByManageStatus(manageStatus);
+ isAcceptPublish = pubSubStatus.getF0();
+ isAcceptSubscribe = pubSubStatus.getF1();
}
BrokerSyncStatusInfo brokerSyncStatusInfo =
brokerSyncStatusInfoMap.get(brokerEntity.getBrokerId());
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
index f2fece8..0e58fbb 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
@@ -19,14 +19,25 @@ package org.apache.tubemq.server.master.web.handler;
import java.util.Date;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.cluster.BrokerInfo;
+import org.apache.tubemq.corebase.cluster.TopicInfo;
import org.apache.tubemq.corebase.utils.SettingValidUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.common.fielddef.WebFieldDef;
import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.common.utils.WebParameterUtils;
import org.apache.tubemq.server.master.TMaster;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbClusterSettingEntity;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbTopicAuthControlEntity;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbTopicConfEntity;
+import org.apache.tubemq.server.master.nodemanage.nodebroker.TopicPSInfoManager;
import org.apache.tubemq.server.master.web.model.ClusterGroupVO;
import org.apache.tubemq.server.master.web.model.ClusterNodeVO;
@@ -51,6 +62,9 @@ public class WebMasterInfoHandler extends AbstractWebHandler {
"getGroupAddressStrInfo");
registerQueryWebMethod("admin_query_cluster_default_setting",
"adminQueryClusterDefSetting");
+ registerQueryWebMethod("admin_query_cluster_topic_view",
+ "adminQueryClusterTopicView");
+
// register modify method
registerModifyWebMethod("admin_transfer_current_master",
"transferCurrentMaster");
@@ -210,7 +224,106 @@ public class WebMasterInfoHandler extends AbstractWebHandler {
return sBuilder;
}
-
+ /**
+ * Query cluster topic overall view
+ *
+ * @param req
+ * @return
+ */
+ public StringBuilder adminQueryClusterTopicView(HttpServletRequest req) {
+ ProcessResult result = new ProcessResult();
+ StringBuilder sBuilder = new StringBuilder(512);
+ // check and get brokerId field
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.COMPSBROKERID, false, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
+ Set<Integer> brokerIds = (Set<Integer>) result.retData1;
+ // check and get topicName field
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
+ Set<String> topicNameSet = (Set<String>) result.retData1;
+ // query topic configure info
+ ConcurrentHashMap<String, List<BdbTopicConfEntity>> topicConfigMap =
+ brokerConfManager.getBdbTopicEntityMap(null);
+ TopicPSInfoManager topicPSInfoManager = master.getTopicPSInfoManager();
+ int totalCount = 0;
+ int brokerCount = 0;
+ int totalCfgNumPartCount = 0;
+ int totalRunNumPartCount = 0;
+ boolean isSrvAcceptPublish = false;
+ boolean isSrvAcceptSubscribe = false;
+ boolean isAcceptPublish = false;
+ boolean isAcceptSubscribe = false;
+ boolean enableAuthControl = false;
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\",\"data\":[");
+ for (Map.Entry<String, List<BdbTopicConfEntity>> entry : topicConfigMap.entrySet()) {
+ if (!topicNameSet.isEmpty() && !topicNameSet.contains(entry.getKey())) {
+ continue;
+ }
+ if (totalCount++ > 0) {
+ sBuilder.append(",");
+ }
+ brokerCount = 0;
+ totalCfgNumPartCount = 0;
+ totalRunNumPartCount = 0;
+ isSrvAcceptPublish = false;
+ isSrvAcceptSubscribe = false;
+ enableAuthControl = false;
+ isAcceptPublish = false;
+ isAcceptSubscribe = false;
+ for (BdbTopicConfEntity entity : entry.getValue()) {
+ if ((!brokerIds.isEmpty()) && (!brokerIds.contains(entity.getBrokerId()))) {
+ continue;
+ }
+ brokerCount++;
+ totalCfgNumPartCount += entity.getNumPartitions() * entity.getNumTopicStores();
+ BdbBrokerConfEntity brokerConfEntity =
+ brokerConfManager.getBrokerDefaultConfigStoreInfo(entity.getBrokerId());
+ if (brokerConfEntity != null) {
+ Tuple2<Boolean, Boolean> pubSubStatus =
+ WebParameterUtils.getPubSubStatusByManageStatus(
+ brokerConfEntity.getManageStatus());
+ isAcceptPublish = pubSubStatus.getF0();
+ isAcceptSubscribe = pubSubStatus.getF1();
+ }
+ BrokerInfo broker =
+ new BrokerInfo(entity.getBrokerId(),
+ entity.getBrokerIp(), entity.getBrokerPort());
+ TopicInfo topicInfo = topicPSInfoManager.getTopicInfo(
+ entity.getTopicName(), broker);
+ if (topicInfo != null) {
+ if (isAcceptPublish && topicInfo.isAcceptPublish()) {
+ isSrvAcceptPublish = true;
+ }
+ if (isAcceptSubscribe && topicInfo.isAcceptSubscribe()) {
+ isSrvAcceptSubscribe = true;
+ }
+ totalRunNumPartCount +=
+ topicInfo.getPartitionNum() * topicInfo.getTopicStoreNum();
+ }
+ }
+ BdbTopicAuthControlEntity authEntity =
+ brokerConfManager.getBdbEnableAuthControlByTopicName(entry.getKey());
+ if (authEntity != null) {
+ enableAuthControl = authEntity.isEnableAuthControl();
+ }
+ sBuilder.append("{\"topicName\":\"").append(entry.getKey())
+ .append("\",\"totalCfgBrokerCnt\":").append(brokerCount)
+ .append(",\"totalCfgNumPart\":").append(totalCfgNumPartCount)
+ .append(",\"totalRunNumPartCount\":").append(totalRunNumPartCount)
+ .append(",\"isSrvAcceptPublish\":").append(isSrvAcceptPublish)
+ .append(",\"isSrvAcceptSubscribe\":").append(isSrvAcceptSubscribe)
+ .append(",\"enableAuthControl\":").append(enableAuthControl)
+ .append("}");
+ }
+ sBuilder.append("],\"dataCount\":").append(totalCount).append("}");
+ return sBuilder;
+ }
}
[incubator-tubemq] 41/49: [TUBEMQ-518] fix parameter pass error
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit a343a3ff19f3ce350db8c92346b69b64d545d196
Author: leno1001 <19...@qq.com>
AuthorDate: Sun Jan 17 11:44:28 2021 +0800
[TUBEMQ-518] fix parameter pass error
Co-authored-by: 曹显乐 <xi...@vivo.com>
---
.../java/org/apache/tubemq/server/common/utils/WebParameterUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index d4b1e20..27e61e1 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -388,7 +388,7 @@ public class WebParameterUtils {
return result.success;
}
checkIntValueNorms(fieldDef, paramValue,
- hasMinVal, minValue, hasMinVal, maxValue, result);
+ hasMinVal, minValue, hasMaxVal, maxValue, result);
}
return result.success;
}
[incubator-tubemq] 16/49: [TUBEMQ-457] There is no need to return
StringBuilder in Master.java (#352)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit bd132b18897b78246af80479382190d24d61873c
Author: Yuanbo Liu <yu...@apache.org>
AuthorDate: Thu Dec 17 15:27:29 2020 +0800
[TUBEMQ-457] There is no need to return StringBuilder in Master.java (#352)
---
.../server/master/web/action/screen/Master.java | 32 +++++++++-------------
1 file changed, 13 insertions(+), 19 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Master.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Master.java
index 7ac535c..60dd31b 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Master.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Master.java
@@ -65,19 +65,19 @@ public class Master implements Action {
}
String type = req.getParameter("type");
if ("consumer".equals(type)) {
- sBuilder = getConsumerListInfo(req, sBuilder);
+ getConsumerListInfo(req, sBuilder);
} else if ("sub_info".equals(type)) {
- sBuilder = getConsumerSubInfo(req, sBuilder);
+ getConsumerSubInfo(req, sBuilder);
} else if ("producer".equals(type)) {
- sBuilder = getProducerListInfo(req, sBuilder);
+ getProducerListInfo(req, sBuilder);
} else if ("broker".equals(type)) {
- sBuilder = innGetBrokerInfo(req, sBuilder, true);
+ innGetBrokerInfo(req, sBuilder, true);
} else if ("newBroker".equals(type)) {
- sBuilder = innGetBrokerInfo(req, sBuilder, false);
+ innGetBrokerInfo(req, sBuilder, false);
} else if ("topic_pub".equals(type)) {
- sBuilder = getTopicPubInfo(req, sBuilder);
+ getTopicPubInfo(req, sBuilder);
} else if ("unbalance_group".equals(type)) {
- sBuilder = getUnbalanceGroupInfo(sBuilder);
+ getUnbalanceGroupInfo(sBuilder);
} else {
sBuilder.append("Unsupported request type : ").append(type);
}
@@ -94,7 +94,7 @@ public class Master implements Action {
* @param sBuilder
* @return
*/
- private StringBuilder getConsumerListInfo(final HttpServletRequest req, StringBuilder sBuilder) {
+ private void getConsumerListInfo(final HttpServletRequest req, StringBuilder sBuilder) {
ConsumerInfoHolder consumerHolder = master.getConsumerHolder();
String group = req.getParameter("group");
if (group != null) {
@@ -115,7 +115,6 @@ public class Master implements Action {
}
}
}
- return sBuilder;
}
/**
@@ -125,7 +124,7 @@ public class Master implements Action {
* @param sBuilder
* @return
*/
- private StringBuilder getConsumerSubInfo(final HttpServletRequest req, StringBuilder sBuilder) {
+ private void getConsumerSubInfo(final HttpServletRequest req, StringBuilder sBuilder) {
ConsumerInfoHolder consumerHolder = master.getConsumerHolder();
String group = req.getParameter("group");
if (group != null) {
@@ -170,7 +169,6 @@ public class Master implements Action {
}
}
}
- return sBuilder;
}
/**
@@ -180,7 +178,7 @@ public class Master implements Action {
* @param sBuilder
* @return
*/
- private StringBuilder getProducerListInfo(final HttpServletRequest req, StringBuilder sBuilder) {
+ private void getProducerListInfo(final HttpServletRequest req, StringBuilder sBuilder) {
String producerId = req.getParameter("id");
if (producerId != null) {
ProducerInfo producer = master.getProducerHolder().getProducerInfo(producerId);
@@ -205,7 +203,6 @@ public class Master implements Action {
}
}
}
- return sBuilder;
}
/**
@@ -216,7 +213,7 @@ public class Master implements Action {
* @param isOldRet
* @return
*/
- private StringBuilder innGetBrokerInfo(final HttpServletRequest req,
+ private void innGetBrokerInfo(final HttpServletRequest req,
StringBuilder sBuilder, boolean isOldRet) {
Map<Integer, BrokerInfo> brokerInfoMap = null;
String brokerIds = req.getParameter("ids");
@@ -269,7 +266,6 @@ public class Master implements Action {
index++;
}
}
- return sBuilder;
}
/**
@@ -279,7 +275,7 @@ public class Master implements Action {
* @param sBuilder
* @return
*/
- private StringBuilder getTopicPubInfo(final HttpServletRequest req, StringBuilder sBuilder) {
+ private void getTopicPubInfo(final HttpServletRequest req, StringBuilder sBuilder) {
String topic = req.getParameter("topic");
Set<String> producerIds = master.getTopicPSInfoManager().getTopicPubInfo(topic);
if (producerIds != null && !producerIds.isEmpty()) {
@@ -289,7 +285,6 @@ public class Master implements Action {
} else {
sBuilder.append("No producer has publish this topic.");
}
- return sBuilder;
}
/**
@@ -298,7 +293,7 @@ public class Master implements Action {
* @param sBuilder
* @return
*/
- private StringBuilder getUnbalanceGroupInfo(StringBuilder sBuilder) {
+ private void getUnbalanceGroupInfo(StringBuilder sBuilder) {
ConsumerInfoHolder consumerHolder = master.getConsumerHolder();
TopicPSInfoManager topicPSInfoManager = master.getTopicPSInfoManager();
Map<String, Map<String, Map<String, Partition>>> currentSubInfoMap =
@@ -326,6 +321,5 @@ public class Master implements Action {
}
}
}
- return sBuilder;
}
}
[incubator-tubemq] 13/49: [TUBEMQ-449]Adjust Example implementation
(#348)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 8acad06b615063659219f8097de0478b76a10011
Author: gosonzhang <46...@qq.com>
AuthorDate: Mon Dec 14 18:21:36 2020 +0800
[TUBEMQ-449]Adjust Example implementation (#348)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../org/apache/tubemq/corebase/utils/Tuple2.java | 53 +++++++
.../tubemq/example/MAMessageProducerExample.java | 158 +++++++++++----------
.../tubemq/example/MessageConsumerExample.java | 60 ++++----
.../tubemq/example/MessageProducerExample.java | 131 +++++++++--------
.../tubemq/example/MessagePullConsumerExample.java | 63 ++++----
.../example/MessagePullSetConsumerExample.java | 62 ++++----
.../tubemq/server/tools/cli/CliProducer.java | 30 ++--
7 files changed, 292 insertions(+), 265 deletions(-)
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java
new file mode 100644
index 0000000..f5626f8
--- /dev/null
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple2.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.corebase.utils;
+
+public class Tuple2<T0, T1> {
+
+ /** Field 0 of the tuple. */
+ public T0 f0 = null;
+ /** Field 1 of the tuple. */
+ public T1 f1 = null;
+
+ /**
+ * Creates a new tuple where all fields are null.
+ */
+ public Tuple2() {
+
+ }
+
+ /**
+ * Creates a new tuple with field 0 specified.
+ *
+ * @param value0 The value for field 0
+ */
+ public Tuple2(T0 value0) {
+ this.f0 = value0;
+ }
+
+ /**
+ * Creates a new tuple and assigns the given values to the tuple's fields.
+ *
+ * @param value0 The value for field 0
+ * @param value1 The value for field 1
+ */
+ public Tuple2(T0 value0, T1 value1) {
+ this.f0 = value0;
+ this.f1 = value1;
+ }
+}
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
index f76b077..ea546c9 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
@@ -20,12 +20,10 @@ package org.apache.tubemq.example;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -41,8 +39,9 @@ import org.apache.tubemq.client.producer.MessageProducer;
import org.apache.tubemq.client.producer.MessageSentCallback;
import org.apache.tubemq.client.producer.MessageSentResult;
import org.apache.tubemq.corebase.Message;
-import org.apache.tubemq.corebase.TErrCodeConstants;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.ThreadUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,19 +51,24 @@ import org.slf4j.LoggerFactory;
* to improve throughput from client to broker.
*/
public class MAMessageProducerExample {
- private static final Logger logger = LoggerFactory.getLogger(MAMessageProducerExample.class);
+ private static final Logger logger =
+ LoggerFactory.getLogger(MAMessageProducerExample.class);
+ private static final AtomicLong TOTAL_COUNTER = new AtomicLong(0);
private static final AtomicLong SENT_SUCC_COUNTER = new AtomicLong(0);
+ private static final AtomicLong SENT_FAIL_COUNTER = new AtomicLong(0);
+ private static final AtomicLong SENT_EXCEPT_COUNTER = new AtomicLong(0);
+
private static final List<MessageProducer> PRODUCER_LIST = new ArrayList<>();
private static final int MAX_PRODUCER_NUM = 100;
private static final int SESSION_FACTORY_NUM = 10;
- private static Set<String> topicSet;
+ private static Map<String, TreeSet<String>> topicAndFiltersMap;
+ private static List<Tuple2<String, String>> topicSendRounds = new ArrayList<>();
private static int msgCount;
- private static int producerCount;
+ private static int clientCount;
private static byte[] sendData;
+ private static AtomicLong filterMsgCount = new AtomicLong(0);
- private final String[] arrayKey = {"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh"};
- private final Set<String> filters = new TreeSet<>();
private final Map<MessageProducer, Sender> producerMap = new HashMap<>();
private final List<MessageSessionFactory> sessionFactoryList = new ArrayList<>();
private final ExecutorService sendExecutorService =
@@ -76,13 +80,9 @@ public class MAMessageProducerExample {
});
private final AtomicInteger producerIndex = new AtomicInteger(0);
- private int keyCount = 0;
- private int sentCount = 0;
- public MAMessageProducerExample(String masterHostAndPort) throws Exception {
- this.filters.add("aaa");
- this.filters.add("bbb");
+ public MAMessageProducerExample(String masterHostAndPort) throws Exception {
TubeClientConfig clientConfig = new TubeClientConfig(masterHostAndPort);
for (int i = 0; i < SESSION_FACTORY_NUM; i++) {
this.sessionFactoryList.add(new TubeMultiSessionFactory(clientConfig));
@@ -90,40 +90,51 @@ public class MAMessageProducerExample {
}
public static void main(String[] args) {
- final String masterHostAndPort = args[0];
-
+ // get call parameters
+ final String masterServers = args[0];
final String topics = args[1];
- final List<String> topicList = Arrays.asList(topics.split(","));
-
- topicSet = new TreeSet<>(topicList);
-
msgCount = Integer.parseInt(args[2]);
- producerCount = Math.min(args.length > 4 ? Integer.parseInt(args[3]) : 10, MAX_PRODUCER_NUM);
-
- logger.info("MAMessageProducerExample.main started...");
-
- final byte[] transmitData = StringUtils.getBytesUtf8("This is a test message from multi-session factory.");
+ clientCount = Math.min(args.length > 4 ? Integer.parseInt(args[3]) : 10, MAX_PRODUCER_NUM);
+ topicAndFiltersMap = MixedUtils.parseTopicParam(topics);
+ // initial topic send round
+ for (Map.Entry<String, TreeSet<String>> entry: topicAndFiltersMap.entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ topicSendRounds.add(new Tuple2<String, String>(entry.getKey()));
+ } else {
+ for (String filter : entry.getValue()) {
+ topicSendRounds.add(new Tuple2<String, String>(entry.getKey(), filter));
+ }
+ }
+ }
+ // build message's body content
+ final byte[] transmitData =
+ StringUtils.getBytesUtf8("This is a test message from multi-session factory.");
final ByteBuffer dataBuffer = ByteBuffer.allocate(1024);
-
while (dataBuffer.hasRemaining()) {
int offset = dataBuffer.arrayOffset();
- dataBuffer.put(transmitData, offset, Math.min(dataBuffer.remaining(), transmitData.length));
+ dataBuffer.put(transmitData, offset,
+ Math.min(dataBuffer.remaining(), transmitData.length));
}
-
dataBuffer.flip();
sendData = dataBuffer.array();
+ // print started log
+ logger.info("MAMessageProducerExample.main started...");
try {
- MAMessageProducerExample messageProducer = new MAMessageProducerExample(masterHostAndPort);
-
+ // initial producer objects
+ MAMessageProducerExample messageProducer =
+ new MAMessageProducerExample(masterServers);
messageProducer.startService();
-
- while (SENT_SUCC_COUNTER.get() < msgCount * producerCount * topicSet.size()) {
- Thread.sleep(1000);
+ // wait util sent message's count reachs required count
+ while (TOTAL_COUNTER.get() < msgCount * clientCount) {
+ logger.info("Sending, total messages is {}, filter messages is {}",
+ SENT_SUCC_COUNTER.get(), filterMsgCount.get());
+ Thread.sleep(5000);
}
+ logger.info("Finished, total messages is {}, filter messages is {}",
+ SENT_SUCC_COUNTER.get(), filterMsgCount.get());
messageProducer.producerMap.clear();
messageProducer.shutdown();
-
} catch (TubeClientException e) {
logger.error("TubeClientException: ", e);
} catch (Throwable e) {
@@ -137,7 +148,7 @@ public class MAMessageProducerExample {
}
private void startService() throws TubeClientException {
- for (int i = 0; i < producerCount; i++) {
+ for (int i = 0; i < clientCount; i++) {
PRODUCER_LIST.add(createProducer());
}
@@ -169,44 +180,44 @@ public class MAMessageProducerExample {
public void run() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm");
try {
- producer.publish(topicSet);
+ producer.publish(topicAndFiltersMap.keySet());
} catch (Throwable t) {
logger.error("publish exception: ", t);
}
- for (int i = 0; i < msgCount; i++) {
+ long sentCount = 0;
+ int roundIndex = 0;
+ int targetCnt = topicSendRounds.size();
+ while (msgCount < 0 || sentCount < msgCount) {
long millis = System.currentTimeMillis();
- for (String topic : topicSet) {
- try {
- Message message = new Message(topic, sendData);
- message.setAttrKeyVal("index", String.valueOf(1));
- message.setAttrKeyVal("dataTime", String.valueOf(millis));
-
- String keyCode = arrayKey[sentCount++ % arrayKey.length];
-
- // date format is accurate to minute, not to second
- message.putSystemHeader(keyCode, sdf.format(new Date(millis)));
- if (filters.contains(keyCode)) {
- keyCount++;
- }
-
- // next line sends message synchronously, which is not recommended
- //producer.sendMessage(message);
-
- // send message asynchronously, recommended
- producer.sendMessage(message, new DefaultSendCallback());
- } catch (Throwable e1) {
- logger.error("sendMessage exception: ", e1);
- }
-
- if (i % 5000 == 0) {
- ThreadUtils.sleep(3000);
- } else if (i % 4000 == 0) {
- ThreadUtils.sleep(2000);
- } else if (i % 2000 == 0) {
- ThreadUtils.sleep(800);
- } else if (i % 1000 == 0) {
- ThreadUtils.sleep(400);
- }
+ roundIndex = (int) (sentCount++ % targetCnt);
+ Tuple2<String, String> target = topicSendRounds.get(roundIndex);
+ Message message = new Message(target.f0, sendData);
+ message.setAttrKeyVal("index", String.valueOf(sentCount));
+ message.setAttrKeyVal("dataTime", String.valueOf(millis));
+ if (target.f1 != null) {
+ filterMsgCount.incrementAndGet();
+ message.putSystemHeader(target.f1, sdf.format(new Date(millis)));
+ }
+ try {
+ // next line sends message synchronously, which is not recommended
+ //producer.sendMessage(message);
+ // send message asynchronously, recommended
+ producer.sendMessage(message, new DefaultSendCallback());
+ } catch (Throwable e1) {
+ TOTAL_COUNTER.incrementAndGet();
+ SENT_EXCEPT_COUNTER.incrementAndGet();
+ logger.error("sendMessage exception: ", e1);
+ }
+ TOTAL_COUNTER.incrementAndGet();
+ // only for test, delay inflight message's count
+ if (sentCount % 5000 == 0) {
+ ThreadUtils.sleep(3000);
+ } else if (sentCount % 4000 == 0) {
+ ThreadUtils.sleep(2000);
+ } else if (sentCount % 2000 == 0) {
+ ThreadUtils.sleep(800);
+ } else if (sentCount % 1000 == 0) {
+ ThreadUtils.sleep(400);
}
}
try {
@@ -221,19 +232,18 @@ public class MAMessageProducerExample {
private class DefaultSendCallback implements MessageSentCallback {
@Override
public void onMessageSent(MessageSentResult result) {
+ TOTAL_COUNTER.incrementAndGet();
if (result.isSuccess()) {
- if (SENT_SUCC_COUNTER.incrementAndGet() % 1000 == 0) {
- logger.info("Send {} message, keyCount is {}", SENT_SUCC_COUNTER.get(), keyCount);
- }
+ SENT_SUCC_COUNTER.incrementAndGet();
} else {
- if (result.getErrCode() != TErrCodeConstants.SERVER_RECEIVE_OVERFLOW) {
- logger.error("Send message failed!" + result.getErrMsg());
- }
+ SENT_FAIL_COUNTER.incrementAndGet();
}
}
@Override
public void onException(Throwable e) {
+ TOTAL_COUNTER.incrementAndGet();
+ SENT_EXCEPT_COUNTER.incrementAndGet();
logger.error("Send message error!", e);
}
}
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
index 0252d16..866be77 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
@@ -17,8 +17,6 @@
package org.apache.tubemq.example;
-import java.util.Arrays;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
@@ -35,6 +33,7 @@ import org.apache.tubemq.client.exception.TubeClientException;
import org.apache.tubemq.client.factory.MessageSessionFactory;
import org.apache.tubemq.client.factory.TubeSingleSessionFactory;
import org.apache.tubemq.corebase.Message;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,59 +54,48 @@ import org.slf4j.LoggerFactory;
*/
public final class MessageConsumerExample {
- private static final Logger logger = LoggerFactory.getLogger(MessageConsumerExample.class);
+ private static final Logger logger =
+ LoggerFactory.getLogger(MessageConsumerExample.class);
private static final MsgRecvStats msgRecvStats = new MsgRecvStats();
private final PushMessageConsumer messageConsumer;
private final MessageSessionFactory messageSessionFactory;
- public MessageConsumerExample(String masterHostAndPort, String group, int fetchCount) throws Exception {
+ public MessageConsumerExample(String masterHostAndPort,
+ String group, int fetchThreadCnt) throws Exception {
ConsumerConfig consumerConfig = new ConsumerConfig(masterHostAndPort, group);
consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_LATEST_OFFSET);
- if (fetchCount > 0) {
- consumerConfig.setPushFetchThreadCnt(fetchCount);
+ if (fetchThreadCnt > 0) {
+ consumerConfig.setPushFetchThreadCnt(fetchThreadCnt);
}
this.messageSessionFactory = new TubeSingleSessionFactory(consumerConfig);
this.messageConsumer = messageSessionFactory.createPushConsumer(consumerConfig);
}
+
public static void main(String[] args) {
- final String masterHostAndPort = args[0];
+ final String masterServers = args[0];
final String topics = args[1];
final String group = args[2];
- final int consumerCount = Integer.parseInt(args[3]);
- int fetchCount = -1;
+ final int clientCount = Integer.parseInt(args[3]);
+ int threadCnt = -1;
if (args.length > 5) {
- fetchCount = Integer.parseInt(args[4]);
- }
- final Map<String, TreeSet<String>> topicTidsMap = new HashMap<>();
-
- String[] topicTidsList = topics.split(",");
- for (String topicTids : topicTidsList) {
- String[] topicTidStr = topicTids.split(":");
- TreeSet<String> tids = null;
- if (topicTidStr.length > 1) {
- String tidsStr = topicTidStr[1];
- String[] tidsSet = tidsStr.split(";");
- if (tidsSet.length > 0) {
- tids = new TreeSet<>(Arrays.asList(tidsSet));
- }
- }
- topicTidsMap.put(topicTidStr[0], tids);
+ threadCnt = Integer.parseInt(args[4]);
}
- final int startFetchCount = fetchCount;
- final ExecutorService executorService = Executors.newCachedThreadPool();
- for (int i = 0; i < consumerCount; i++) {
+ final int fetchThreadCnt = threadCnt;
+ final Map<String, TreeSet<String>> topicAndFiltersMap =
+ MixedUtils.parseTopicParam(topics);
+ final ExecutorService executorService =
+ Executors.newCachedThreadPool();
+ for (int i = 0; i < clientCount; i++) {
executorService.submit(new Runnable() {
@Override
public void run() {
try {
- MessageConsumerExample messageConsumer = new MessageConsumerExample(
- masterHostAndPort,
- group,
- startFetchCount
- );
- messageConsumer.subscribe(topicTidsMap);
+ MessageConsumerExample messageConsumer =
+ new MessageConsumerExample(masterServers,
+ group, fetchThreadCnt);
+ messageConsumer.subscribe(topicAndFiltersMap);
} catch (Exception e) {
logger.error("Create consumer failed!", e);
}
@@ -126,8 +114,8 @@ public final class MessageConsumerExample {
msgRecvStats.stopStats();
}
- public void subscribe(Map<String, TreeSet<String>> topicTidsMap) throws TubeClientException {
- for (Map.Entry<String, TreeSet<String>> entry : topicTidsMap.entrySet()) {
+ public void subscribe(Map<String, TreeSet<String>> topicAndFiltersMap) throws TubeClientException {
+ for (Map.Entry<String, TreeSet<String>> entry : topicAndFiltersMap.entrySet()) {
MessageV2Listener messageListener = new DefaultMessageListener(entry.getKey());
messageConsumer.subscribe(entry.getKey(), entry.getValue(), messageListener);
}
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
index 57d0ef0..7c29a63 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageProducerExample.java
@@ -19,7 +19,7 @@ package org.apache.tubemq.example;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
-import java.util.Arrays;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -36,7 +36,9 @@ import org.apache.tubemq.client.producer.MessageProducer;
import org.apache.tubemq.client.producer.MessageSentCallback;
import org.apache.tubemq.client.producer.MessageSentResult;
import org.apache.tubemq.corebase.Message;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.ThreadUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,32 +50,40 @@ import org.slf4j.LoggerFactory;
*/
public final class MessageProducerExample {
- private static final Logger logger = LoggerFactory.getLogger(MessageProducerExample.class);
- private static final ConcurrentHashMap<String, AtomicLong> counterMap = new ConcurrentHashMap<>();
+ private static final Logger logger =
+ LoggerFactory.getLogger(MessageProducerExample.class);
+ private static final ConcurrentHashMap<String, AtomicLong> counterMap =
+ new ConcurrentHashMap<>();
- private final String[] arrayKey = {"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh"};
- private final Set<String> filters = new TreeSet<>();
private final MessageProducer messageProducer;
private final MessageSessionFactory messageSessionFactory;
- private int keyCount = 0;
- private int sentCount = 0;
-
- public MessageProducerExample(String masterHostAndPort) throws Exception {
- filters.add("aaa");
- filters.add("bbb");
-
- TubeClientConfig clientConfig = new TubeClientConfig(masterHostAndPort);
+ public MessageProducerExample(String masterServers) throws Exception {
+ TubeClientConfig clientConfig = new TubeClientConfig(masterServers);
this.messageSessionFactory = new TubeSingleSessionFactory(clientConfig);
this.messageProducer = messageSessionFactory.createProducer();
}
public static void main(String[] args) {
- final String masterHostAndPort = args[0];
+ // get and initial parameters
+ final String masterServers = args[0];
final String topics = args[1];
- final List<String> topicList = Arrays.asList(topics.split(","));
- final int count = Integer.parseInt(args[2]);
-
+ final long msgCount = Long.parseLong(args[2]);
+ final Map<String, TreeSet<String>> topicAndFiltersMap =
+ MixedUtils.parseTopicParam(topics);
+ // initial send target
+ final List<Tuple2<String, String>> topicSendRounds = new ArrayList<>();
+ // initial topic send round
+ for (Map.Entry<String, TreeSet<String>> entry: topicAndFiltersMap.entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ topicSendRounds.add(new Tuple2<String, String>(entry.getKey()));
+ } else {
+ for (String filter : entry.getValue()) {
+ topicSendRounds.add(new Tuple2<String, String>(entry.getKey(), filter));
+ }
+ }
+ }
+ // initial sent data
String body = "This is a test message from single-session-factory.";
byte[] bodyBytes = StringUtils.getBytesUtf8(body);
final ByteBuffer dataBuffer = ByteBuffer.allocate(1024);
@@ -82,32 +92,40 @@ public final class MessageProducerExample {
dataBuffer.put(bodyBytes, offset, Math.min(dataBuffer.remaining(), bodyBytes.length));
}
dataBuffer.flip();
+ // send messages
try {
- MessageProducerExample messageProducer = new MessageProducerExample(masterHostAndPort);
- messageProducer.publishTopics(topicList);
- for (int i = 0; i < count; i++) {
- for (String topic : topicList) {
- try {
- // next line sends message synchronously, which is not recommended
- // messageProducer.sendMessage(topic, body.getBytes());
-
- // send message asynchronously, recommended
- messageProducer.sendMessageAsync(
- i,
- topic,
- dataBuffer.array(),
- messageProducer.new DefaultSendCallback()
- );
- } catch (Throwable e1) {
- logger.error("Send Message throw exception ", e1);
- }
+ long sentCount = 0;
+ int roundIndex = 0;
+ int targetCnt = topicSendRounds.size();
+ MessageProducerExample messageProducer =
+ new MessageProducerExample(masterServers);
+ messageProducer.publishTopics(topicAndFiltersMap.keySet());
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm");
+ while (msgCount < 0 || sentCount < msgCount) {
+ roundIndex = (int) (sentCount++ % targetCnt);
+ Tuple2<String, String> target = topicSendRounds.get(roundIndex);
+ Message message = new Message(target.f0, body.getBytes());
+ long currTimeMillis = System.currentTimeMillis();
+ message.setAttrKeyVal("index", String.valueOf(sentCount));
+ message.setAttrKeyVal("dataTime", String.valueOf(currTimeMillis));
+ if (target.f1 != null) {
+ message.putSystemHeader(target.f1, sdf.format(new Date(currTimeMillis)));
}
-
- if (i % 20000 == 0) {
+ try {
+ // 1.1 next line sends message synchronously, which is not recommended
+ // messageProducer.sendMessage(message);
+ // 1.2 send message asynchronously, recommended
+ messageProducer.sendMessageAsync(message,
+ messageProducer.new DefaultSendCallback());
+ } catch (Throwable e1) {
+ logger.error("Send Message throw exception ", e1);
+ }
+ // only for test, delay inflight message's count
+ if (sentCount % 20000 == 0) {
ThreadUtils.sleep(4000);
- } else if (i % 10000 == 0) {
+ } else if (sentCount % 10000 == 0) {
ThreadUtils.sleep(2000);
- } else if (i % 2500 == 0) {
+ } else if (sentCount % 2500 == 0) {
ThreadUtils.sleep(300);
}
}
@@ -124,51 +142,33 @@ public final class MessageProducerExample {
}
}
- public void publishTopics(List<String> topicList) throws TubeClientException {
- this.messageProducer.publish(new TreeSet<>(topicList));
+ public void publishTopics(Set<String> topicSet) throws TubeClientException {
+ this.messageProducer.publish(topicSet);
}
/**
* Send message synchronous.
*/
- public void sendMessage(String topic, byte[] body) {
+ public void sendMessage(Message message) {
// date format is accurate to minute, not to second
- SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm");
- long currTimeMillis = System.currentTimeMillis();
- Message message = new Message(topic, body);
- message.setAttrKeyVal("index", String.valueOf(1));
- message.setAttrKeyVal("dataTime", String.valueOf(currTimeMillis));
- message.putSystemHeader("test", sdf.format(new Date(currTimeMillis)));
try {
MessageSentResult result = messageProducer.sendMessage(message);
if (!result.isSuccess()) {
- logger.error("Send message failed!" + result.getErrMsg());
+ logger.error("Sync-send message failed!" + result.getErrMsg());
}
} catch (TubeClientException | InterruptedException e) {
- logger.error("Send message failed!", e);
+ logger.error("Sync-send message failed!", e);
}
}
/**
* Send message asynchronous. More efficient and recommended.
*/
- public void sendMessageAsync(int id, String topic, byte[] body, MessageSentCallback callback) {
- Message message = new Message(topic, body);
-
- // date format is accurate to minute, not to second
- SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm");
- long currTimeMillis = System.currentTimeMillis();
- message.setAttrKeyVal("index", String.valueOf(1));
- String keyCode = arrayKey[sentCount++ % arrayKey.length];
- message.putSystemHeader(keyCode, sdf.format(new Date(currTimeMillis)));
- if (filters.contains(keyCode)) {
- keyCount++;
- }
+ public void sendMessageAsync(Message message, MessageSentCallback callback) {
try {
- message.setAttrKeyVal("dataTime", String.valueOf(currTimeMillis));
messageProducer.sendMessage(message, callback);
} catch (TubeClientException | InterruptedException e) {
- logger.error("Send message failed!", e);
+ logger.error("Async-send message failed!", e);
}
}
@@ -187,9 +187,8 @@ public final class MessageProducerExample {
currCount = tmpCount;
}
}
-
if (currCount.incrementAndGet() % 1000 == 0) {
- logger.info("Send " + topicName + " " + currCount.get() + " message, keyCount is " + keyCount);
+ logger.info("Send " + topicName + " " + currCount.get() + " message!");
}
} else {
logger.error("Send message failed!" + result.getErrMsg());
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullConsumerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullConsumerExample.java
index 16c45b2..36fcaa2 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullConsumerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullConsumerExample.java
@@ -17,9 +17,10 @@
package org.apache.tubemq.example;
-import java.util.Arrays;
+import static org.apache.tubemq.corebase.TErrCodeConstants.IGNORE_ERROR_SET;
import java.util.List;
import java.util.Map;
+import java.util.TreeSet;
import org.apache.tubemq.client.config.ConsumerConfig;
import org.apache.tubemq.client.consumer.ConsumeOffsetInfo;
import org.apache.tubemq.client.consumer.ConsumePosition;
@@ -29,9 +30,11 @@ import org.apache.tubemq.client.exception.TubeClientException;
import org.apache.tubemq.client.factory.MessageSessionFactory;
import org.apache.tubemq.client.factory.TubeSingleSessionFactory;
import org.apache.tubemq.corebase.Message;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* This demo shows how to consume message by pull.
*
@@ -42,7 +45,8 @@ import org.slf4j.LoggerFactory;
*/
public final class MessagePullConsumerExample {
- private static final Logger logger = LoggerFactory.getLogger(MessagePullConsumerExample.class);
+ private static final Logger logger =
+ LoggerFactory.getLogger(MessagePullConsumerExample.class);
private static final MsgRecvStats msgRecvStats = new MsgRecvStats();
private final PullMessageConsumer messagePullConsumer;
@@ -56,39 +60,37 @@ public final class MessagePullConsumerExample {
}
public static void main(String[] args) throws Throwable {
- final String masterHostAndPort = args[0];
+ // get and initial parameters
+ final String masterServers = args[0];
final String topics = args[1];
final String group = args[2];
- final int consumeCount = Integer.parseInt(args[3]);
-
- final MessagePullConsumerExample messageConsumer = new MessagePullConsumerExample(
- masterHostAndPort,
- group
- );
-
- final List<String> topicList = Arrays.asList(topics.split(","));
- messageConsumer.subscribe(topicList);
- long startTime = System.currentTimeMillis();
-
+ final int msgCount = Integer.parseInt(args[3]);
+ final Map<String, TreeSet<String>> topicAndFiltersMap =
+ MixedUtils.parseTopicParam(topics);
+ // initial consumer object
+ final MessagePullConsumerExample messageConsumer =
+ new MessagePullConsumerExample(masterServers, group);
+ messageConsumer.subscribe(topicAndFiltersMap);
Thread[] fetchRunners = new Thread[3];
for (int i = 0; i < fetchRunners.length; i++) {
- fetchRunners[i] = new Thread(new FetchRequestRunner(messageConsumer, consumeCount));
+ fetchRunners[i] = new Thread(new FetchRequestRunner(messageConsumer, msgCount));
fetchRunners[i].setName("_fetch_runner_" + i);
}
-
+ // initial fetch threads
for (Thread thread : fetchRunners) {
thread.start();
}
-
- Thread statisticThread = new Thread(msgRecvStats, "Sent Statistic Thread");
+ // initial statistic thread
+ Thread statisticThread =
+ new Thread(msgRecvStats, "Sent Statistic Thread");
statisticThread.start();
}
- public void subscribe(List<String> topicList) throws TubeClientException {
- for (String topic : topicList) {
- messagePullConsumer.subscribe(topic, null);
+ public void subscribe(
+ Map<String, TreeSet<String>> topicAndFiltersMap) throws TubeClientException {
+ for (Map.Entry<String, TreeSet<String>> entry : topicAndFiltersMap.entrySet()) {
+ messagePullConsumer.subscribe(entry.getKey(), entry.getValue());
}
-
messagePullConsumer.completeSubscribe();
}
@@ -96,7 +98,8 @@ public final class MessagePullConsumerExample {
return messagePullConsumer.getMessage();
}
- public ConsumerResult confirmConsume(final String confirmContext, boolean isConsumed) throws TubeClientException {
+ public ConsumerResult confirmConsume(final String confirmContext,
+ boolean isConsumed) throws TubeClientException {
return messagePullConsumer.confirmConsume(confirmContext, isConsumed);
}
@@ -109,9 +112,9 @@ public final class MessagePullConsumerExample {
final MessagePullConsumerExample messageConsumer;
final int consumeCount;
- FetchRequestRunner(final MessagePullConsumerExample messageConsumer, int count) {
+ FetchRequestRunner(final MessagePullConsumerExample messageConsumer, int msgCount) {
this.messageConsumer = messageConsumer;
- this.consumeCount = count;
+ this.consumeCount = msgCount;
}
@Override
@@ -127,16 +130,10 @@ public final class MessagePullConsumerExample {
}
messageConsumer.confirmConsume(result.getConfirmContext(), true);
} else {
- if (!(result.getErrCode() == 400
- || result.getErrCode() == 404
- || result.getErrCode() == 405
- || result.getErrCode() == 406
- || result.getErrCode() == 407
- || result.getErrCode() == 408)) {
+ if (!IGNORE_ERROR_SET.contains(result.getErrCode())) {
logger.info(
"Receive messages errorCode is {}, Error message is {}",
- result.getErrCode(),
- result.getErrMsg());
+ result.getErrCode(), result.getErrMsg());
}
}
if (consumeCount > 0) {
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullSetConsumerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullSetConsumerExample.java
index d3afeaa..3d49478 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullSetConsumerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessagePullSetConsumerExample.java
@@ -17,7 +17,7 @@
package org.apache.tubemq.example;
-import java.util.Arrays;
+import static org.apache.tubemq.corebase.TErrCodeConstants.IGNORE_ERROR_SET;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
@@ -34,9 +34,11 @@ import org.apache.tubemq.client.exception.TubeClientException;
import org.apache.tubemq.client.factory.MessageSessionFactory;
import org.apache.tubemq.client.factory.TubeSingleSessionFactory;
import org.apache.tubemq.corebase.Message;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* This demo shows how to reset offset on consuming. The main difference from {@link MessagePullConsumerExample}
* is that we call {@link PullMessageConsumer#completeSubscribe(String, int, boolean, Map)} instead of
@@ -45,24 +47,32 @@ import org.slf4j.LoggerFactory;
*/
public final class MessagePullSetConsumerExample {
- private static final Logger logger = LoggerFactory.getLogger(MessagePullSetConsumerExample.class);
+ private static final Logger logger =
+ LoggerFactory.getLogger(MessagePullSetConsumerExample.class);
private static final AtomicLong counter = new AtomicLong(0);
private final PullMessageConsumer messagePullConsumer;
private final MessageSessionFactory messageSessionFactory;
- public MessagePullSetConsumerExample(String masterHostAndPort, String group) throws Exception {
+ public MessagePullSetConsumerExample(String masterHostAndPort,
+ String group) throws Exception {
ConsumerConfig consumerConfig = new ConsumerConfig(masterHostAndPort, group);
this.messageSessionFactory = new TubeSingleSessionFactory(consumerConfig);
this.messagePullConsumer = messageSessionFactory.createPullConsumer(consumerConfig);
}
public static void main(String[] args) {
- final String masterHostAndPort = args[0];
+ // get and initial parameters
+ final String masterServers = args[0];
final String topics = args[1];
final String group = args[2];
- final int consumeCount = Integer.parseInt(args[3]);
- final Map<String, Long> partOffsetMap = new ConcurrentHashMap<>();
+ final int msgCount = Integer.parseInt(args[3]);
+ final Map<String, TreeSet<String>> topicAndFiltersMap =
+ MixedUtils.parseTopicParam(topics);
+ // initial reset offset parameters
+ // (The offset specified is only a demo)
+ final Map<String, Long> partOffsetMap =
+ new ConcurrentHashMap<>();
partOffsetMap.put("123:test_1:0", 0L);
partOffsetMap.put("123:test_1:1", 0L);
partOffsetMap.put("123:test_1:2", 0L);
@@ -70,17 +80,15 @@ public final class MessagePullSetConsumerExample {
partOffsetMap.put("123:test_2:1", 350L);
partOffsetMap.put("123:test_2:2", 350L);
- final List<String> topicList = Arrays.asList(topics.split(","));
-
ExecutorService executorService = Executors.newCachedThreadPool();
executorService.submit(new Runnable() {
@Override
public void run() {
try {
- int getCount = consumeCount;
+ int getCount = msgCount;
MessagePullSetConsumerExample messageConsumer =
- new MessagePullSetConsumerExample(masterHostAndPort, group);
- messageConsumer.subscribe(topicList, partOffsetMap);
+ new MessagePullSetConsumerExample(masterServers, group);
+ messageConsumer.subscribe(topicAndFiltersMap, partOffsetMap);
// main logic of consuming
do {
ConsumerResult result = messageConsumer.getMessage();
@@ -125,19 +133,13 @@ public final class MessagePullSetConsumerExample {
confirmResult.getErrMsg());
}
} else {
- if (!(result.getErrCode() == 400
- || result.getErrCode() == 404
- || result.getErrCode() == 405
- || result.getErrCode() == 406
- || result.getErrCode() == 407
- || result.getErrCode() == 408)) {
+ if (!IGNORE_ERROR_SET.contains(result.getErrCode())) {
logger.info(
"Receive messages errorCode is {}, Error message is {}",
- result.getErrCode(),
- result.getErrMsg());
+ result.getErrCode(), result.getErrMsg());
}
}
- if (consumeCount >= 0) {
+ if (msgCount >= 0) {
if (--getCount <= 0) {
break;
}
@@ -157,24 +159,16 @@ public final class MessagePullSetConsumerExample {
}
}
- public void subscribe(
- List<String> topicList,
- Map<String, Long> partOffsetMap
- ) throws TubeClientException {
- TreeSet<String> filters = new TreeSet<>();
- filters.add("aaa");
- filters.add("bbb");
- for (String topic : topicList) {
- this.messagePullConsumer.subscribe(topic, filters);
+ public void subscribe(Map<String, TreeSet<String>> topicAndFiltersMap,
+ Map<String, Long> partOffsetMap) throws TubeClientException {
+ for (Map.Entry<String, TreeSet<String>> entry : topicAndFiltersMap.entrySet()) {
+ messagePullConsumer.subscribe(entry.getKey(), entry.getValue());
}
String sessionKey = "test_reset2";
int consumerCount = 2;
boolean isSelectBig = false;
- messagePullConsumer.completeSubscribe(
- sessionKey,
- consumerCount,
- isSelectBig,
- partOffsetMap);
+ messagePullConsumer.completeSubscribe(sessionKey,
+ consumerCount, isSelectBig, partOffsetMap);
}
public ConsumerResult getMessage() throws TubeClientException {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
index 1b2f25a..f544829 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
@@ -42,6 +42,7 @@ import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corebase.utils.ThreadUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.common.fielddef.CliArgDef;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +65,7 @@ public class CliProducer extends CliAbstractBase {
// sent data content
private static byte[] sentData;
private final Map<String, TreeSet<String>> topicAndFiltersMap = new HashMap<>();
- private final List<TupleValue> topicSendRounds = new ArrayList<>();
+ private final List<Tuple2<String, String>> topicSendRounds = new ArrayList<>();
private final List<MessageSessionFactory> sessionFactoryList = new ArrayList<>();
private final Map<MessageProducer, MsgSender> producerMap = new HashMap<>();
// cli parameters
@@ -189,10 +190,10 @@ public class CliProducer extends CliAbstractBase {
// initial topic send round
for (Map.Entry<String, TreeSet<String>> entry: topicAndFiltersMap.entrySet()) {
if (entry.getValue().isEmpty()) {
- topicSendRounds.add(new TupleValue(entry.getKey()));
+ topicSendRounds.add(new Tuple2<String, String>(entry.getKey()));
} else {
for (String filter : entry.getValue()) {
- topicSendRounds.add(new TupleValue(entry.getKey(), filter));
+ topicSendRounds.add(new Tuple2<String, String>(entry.getKey(), filter));
}
}
}
@@ -266,11 +267,11 @@ public class CliProducer extends CliAbstractBase {
roundIndex = (int) (sentCount++ % topicAndCondCnt);
try {
long millis = System.currentTimeMillis();
- TupleValue tupleValue = topicSendRounds.get(roundIndex);
- Message message = new Message(tupleValue.topic, sentData);
- if (tupleValue.filter != null) {
+ Tuple2<String, String> target = topicSendRounds.get(roundIndex);
+ Message message = new Message(target.f0, sentData);
+ if (target.f1 != null) {
// if include filter, add filter item
- message.putSystemHeader(tupleValue.filter, sdf.format(new Date(millis)));
+ message.putSystemHeader(target.f1, sdf.format(new Date(millis)));
}
// use sync or async process
if (syncProduction) {
@@ -332,21 +333,6 @@ public class CliProducer extends CliAbstractBase {
}
}
- private static class TupleValue {
- public String topic = null;
- public String filter = null;
-
- public TupleValue(String topic) {
- this.topic = topic;
- }
-
- public TupleValue(String topic, String filter) {
- this.topic = topic;
- this.filter = filter;
- }
-
- }
-
public static void main(String[] args) {
CliProducer cliProducer = new CliProducer();
try {
[incubator-tubemq] 30/49: [TUBEMQ-500] Add setting operate API
(#389)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 20c4602477b096eb14058fa067bfc6e1e8f6ca5f
Author: gosonzhang <46...@qq.com>
AuthorDate: Mon Jan 11 10:17:54 2021 +0800
[TUBEMQ-500] Add setting operate API (#389)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../tubemq/client/producer/AllowedSetting.java | 61 ++++++++
.../org/apache/tubemq/corebase/TBaseConstants.java | 1 +
.../apache/tubemq/corebase/utils/MixedUtils.java | 9 --
.../tubemq/corebase/utils/SettingValidUtils.java | 39 +++++
.../tubemq/server/common/fielddef/WebFieldDef.java | 14 +-
.../server/common/utils/WebParameterUtils.java | 165 ++++++++++++++-------
.../bdbentitys/BdbClusterSettingEntity.java | 17 +--
.../nodemanage/nodebroker/BrokerConfManager.java | 62 ++------
.../master/web/handler/WebMasterInfoHandler.java | 119 ++++++++++++++-
9 files changed, 353 insertions(+), 134 deletions(-)
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java
new file mode 100644
index 0000000..cabf928
--- /dev/null
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/producer/AllowedSetting.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.client.producer;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.protobuf.generated.ClientMaster;
+import org.apache.tubemq.corebase.utils.SettingValidUtils;
+
+/**
+ * The class class caches the dynamic settings
+ * returned from the server.
+ */
+public class AllowedSetting {
+ private AtomicLong configId =
+ new AtomicLong(TBaseConstants.META_VALUE_UNDEFINED);
+ private AtomicInteger maxMsgSize =
+ new AtomicInteger(TBaseConstants.META_MAX_MESSAGE_DATA_SIZE);
+
+ public AllowedSetting() {
+
+ }
+
+ // set master returned configure
+ public void updAllowedSetting(ClientMaster.ApprovedClientConfig allowedConfig) {
+ if (allowedConfig != null) {
+ if (configId.get() != allowedConfig.getConfigId()) {
+ configId.set(allowedConfig.getConfigId());
+ }
+ if (allowedConfig.hasMaxMsgSize()
+ && allowedConfig.getMaxMsgSize() != maxMsgSize.get()) {
+ maxMsgSize.set(
+ SettingValidUtils.validAndGetMaxMsgSize(allowedConfig.getMaxMsgSize()));
+ }
+ }
+ }
+
+ public long getConfigId() {
+ return configId.get();
+ }
+
+ public int getMaxMsgSize() {
+ return maxMsgSize.get();
+ }
+}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java
index 2f1be7a..d91b083 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/TBaseConstants.java
@@ -31,6 +31,7 @@ public class TBaseConstants {
public static final int META_MAX_MSGTYPE_LENGTH = 255;
public static final int META_MAX_MESSAGE_HEADER_SIZE = 1024;
public static final int META_MAX_MESSAGE_DATA_SIZE = 1024 * 1024;
+ public static final int META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT = 20 * 1024 * 1024;
public static final int META_MAX_PARTITION_COUNT = 100;
public static final int META_MAX_BROKER_IP_LENGTH = 32;
public static final int META_MAX_USERNAME_LENGTH = 64;
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
index bcd0738..bfbedad 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
@@ -92,13 +92,4 @@ public class MixedUtils {
dataBuffer.flip();
return dataBuffer.array();
}
-
- // get the middle data between min, max, and data
- public static int mid(int data, int min, int max) {
- return Math.max(min, Math.min(max, data));
- }
-
- public static long mid(long data, long min, long max) {
- return Math.max(min, Math.min(max, data));
- }
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java
new file mode 100644
index 0000000..4a206ef
--- /dev/null
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/SettingValidUtils.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.corebase.utils;
+
+import org.apache.tubemq.corebase.TBaseConstants;
+
+
+public class SettingValidUtils {
+
+ // get the middle data between min, max, and data
+ public static int mid(int data, int min, int max) {
+ return Math.max(min, Math.min(max, data));
+ }
+
+ public static long mid(long data, long min, long max) {
+ return Math.max(min, Math.min(max, data));
+ }
+
+ public static int validAndGetMaxMsgSize(int inMaxMsgSize) {
+ return mid(inMaxMsgSize,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT);
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index a65a223..05688a7 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -84,7 +84,19 @@ public enum WebFieldDef {
OFFSETJSON(21, "offsetJsonInfo", "offsetInfo",
WebFieldType.JSONTYPE, "The offset info that needs to be added or modified"),
ONLYMEM(22, "onlyMemory", "onlyMem", WebFieldType.BOOLEAN,
- "Only clear the offset data in the memory cache, default is false");
+ "Only clear the offset data in the memory cache, default is false"),
+ ADMINAUTHTOKEN(23, "confModAuthToken", "authToken", WebFieldType.STRING,
+ "Admin api operation authorization code",
+ TServerConstants.CFG_MODAUTHTOKEN_MAX_LENGTH),
+ MAXMSGSIZE(24, "maxMsgSize", "maxMsgSize", WebFieldType.INT,
+ "Max allowed message size", RegexDef.TMP_NUMBER),
+ CREATEDATE(25, "createDate", "cDate", WebFieldType.STRING,
+ "Record creation date", TBaseConstants.META_MAX_DATEVALUE_LENGTH),
+ MODIFYDATE(26, "modifyDate", "mDate", WebFieldType.STRING,
+ "Record modification date", TBaseConstants.META_MAX_DATEVALUE_LENGTH);
+
+
+
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index f309ab7..385fe22 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -246,6 +246,10 @@ public class WebParameterUtils {
return strBuffer.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
}
+ public static StringBuilder buildSuccessResult(StringBuilder strBuffer, String appendInfo) {
+ return strBuffer.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"").
+ append(appendInfo).append("\"}");
+ }
/**
* Parse the parameter value from an object value to a long value
*
@@ -273,10 +277,9 @@ public class WebParameterUtils {
long paramIntVal = Long.parseLong(paramValue);
result.setSuccResult(paramIntVal);
} catch (Throwable e) {
- result.setFailResult(400,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name).append(" parse error: ")
- .append(e.getMessage()).toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" parse error: ").append(e.getMessage()).toString());
}
return result.success;
}
@@ -429,10 +432,9 @@ public class WebParameterUtils {
// Check if the parameter exists
if (TStringUtils.isBlank(paramValue)) {
if (required) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append(" is missing or value is null or blank!").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" is missing or value is null or blank!").toString());
} else {
procStringDefValue(fieldDef.isCompFieldType(), defValue, result);
}
@@ -455,10 +457,9 @@ public class WebParameterUtils {
// check if is empty result
if (valItemSet.isEmpty()) {
if (required) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append(" is missing or value is null or blank!").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" is missing or value is null or blank!").toString());
} else {
procStringDefValue(fieldDef.isCompFieldType(), defValue, result);
}
@@ -467,11 +468,10 @@ public class WebParameterUtils {
// check max item count
if (fieldDef.itemMaxCnt != TBaseConstants.META_VALUE_UNDEFINED) {
if (valItemSet.size() > fieldDef.itemMaxCnt) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append("'s item count over max allowed count (")
- .append(fieldDef.itemMaxCnt).append(")!").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append("'s item count over max allowed count (")
+ .append(fieldDef.itemMaxCnt).append(")!").toString());
}
}
result.setSuccResult(valItemSet);
@@ -511,10 +511,9 @@ public class WebParameterUtils {
// Check if the parameter exists
if (TStringUtils.isBlank(paramValue)) {
if (required) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append(" is missing or value is null or blank!").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" is missing or value is null or blank!").toString());
} else {
result.setSuccResult(defValue);
}
@@ -524,18 +523,15 @@ public class WebParameterUtils {
paramValue = URLDecoder.decode(paramValue,
TBaseConstants.META_DEFAULT_CHARSET_NAME);
} catch (UnsupportedEncodingException e) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append(" decode error, exception is ")
- .append(e.toString()).toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" decode error, exception is ")
+ .append(e.toString()).toString());
}
if (TStringUtils.isBlank(paramValue)) {
if (required) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append("'s value is blank!").toString());
+ result.setFailResult(new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name).append("'s value is blank!").toString());
} else {
result.setSuccResult(defValue);
}
@@ -543,11 +539,10 @@ public class WebParameterUtils {
}
if (fieldDef.valMaxLen != TBaseConstants.META_VALUE_UNDEFINED) {
if (paramValue.length() > fieldDef.valMaxLen) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append("'s length over max allowed length (")
- .append(fieldDef.valMaxLen).append(")!").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append("'s length over max allowed length (")
+ .append(fieldDef.valMaxLen).append(")!").toString());
return result.success;
}
}
@@ -557,11 +552,71 @@ public class WebParameterUtils {
new TypeToken<Map<String, Long>>(){}.getType());
result.setSuccResult(manOffsets);
} catch (Throwable e) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name)
- .append(" value parse failure, error is ")
- .append(e.getMessage()).append("!").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" value parse failure, error is ")
+ .append(e.getMessage()).append("!").toString());
+ }
+ return result.success;
+ }
+
+ /**
+ * Parse the parameter value from an string value to Date value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if failed to parse value from the given object
+ * @param result process result
+ * @return valid result for the parameter value
+ */
+ public static boolean getDateParameter(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ Date defValue,
+ ProcessResult result) {
+ if (!getStringParamValue(req, fieldDef, required, null, result)) {
+ return result.success;
+ }
+ String paramValue = (String) result.retData1;
+ if (paramValue == null) {
+ result.setSuccResult(defValue);
+ return result.success;
+ }
+ try {
+ DateFormat sdf = new SimpleDateFormat(TBaseConstants.META_TMP_DATE_VALUE);
+ Date date = sdf.parse(paramValue);
+ result.setSuccResult(date);
+ } catch (Throwable e) {
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" parse error: ").append(e.getMessage()).toString());
+ }
+ return result.success;
+ }
+
+ /**
+ * Valid execution authorization info
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param master current master object
+ * @param result process result
+ * @return valid result for the parameter value
+ */
+ public static boolean validReqAuthorizeInfo(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ TMaster master,
+ ProcessResult result) {
+ if (!getStringParamValue(req, fieldDef, required, null, result)) {
+ return result.success;
+ }
+ String paramValue = (String) result.retData1;
+ if (paramValue != null) {
+ if (!paramValue.equals(master.getMasterConfig().getConfModAuthToken())) {
+ result.setFailResult("Illegal access, unauthorized request!");
+ }
}
return result.success;
}
@@ -608,20 +663,20 @@ public class WebParameterUtils {
// check value's max length
if (fieldDef.valMaxLen != TBaseConstants.META_VALUE_UNDEFINED) {
if (paramVal.length() > fieldDef.valMaxLen) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("over max length for ")
- .append(fieldDef.name).append(", only allow ")
- .append(fieldDef.valMaxLen).append(" length").toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("over max length for ").append(fieldDef.name)
+ .append(", only allow ").append(fieldDef.valMaxLen)
+ .append(" length").toString());
return false;
}
}
// check value's pattern
if (fieldDef.regexCheck) {
if (!paramVal.matches(fieldDef.regexDef.getPattern())) {
- result.setFailResult(fieldDef.id,
- new StringBuilder(512).append("illegal value for ")
- .append(fieldDef.name).append(", value ")
- .append(fieldDef.regexDef.getErrMsgTemp()).toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("illegal value for ").append(fieldDef.name)
+ .append(", value ").append(fieldDef.regexDef.getErrMsgTemp())
+ .toString());
return false;
}
}
@@ -647,18 +702,16 @@ public class WebParameterUtils {
try {
int paramIntVal = Integer.parseInt(paramValue);
if (hasMinVal && paramIntVal < minValue) {
- result.setFailResult(400,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name).append(" value must >= ")
- .append(minValue).toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" value must >= ").append(minValue).toString());
return false;
}
result.setSuccResult(paramIntVal);
} catch (Throwable e) {
- result.setFailResult(400,
- new StringBuilder(512).append("Parameter ")
- .append(fieldDef.name).append(" parse error: ")
- .append(e.getMessage()).toString());
+ result.setFailResult(new StringBuilder(512)
+ .append("Parameter ").append(fieldDef.name)
+ .append(" parse error: ").append(e.getMessage()).toString());
return false;
}
return true;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
index ca6e1b4..588fd87 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
@@ -33,7 +33,7 @@ import org.apache.tubemq.server.common.utils.WebParameterUtils;
@Entity
public class BdbClusterSettingEntity implements Serializable {
- private static final long serialVersionUID = -3259439355290322115L;
+ private static final long serialVersionUID = 3259439355290322115L;
@PrimaryKey
private String recordKey = "";
@@ -62,7 +62,7 @@ public class BdbClusterSettingEntity implements Serializable {
private String deleteWhen = ""; //delete policy execute time
private int qryPriorityId = TBaseConstants.META_VALUE_UNDEFINED;
private int maxMsgSize = TBaseConstants.META_VALUE_UNDEFINED;
- private String attributes; //extra attribute
+ private String attributes = ""; //extra attribute
private String modifyUser; //modify user
private Date modifyDate; //modify date
@@ -235,22 +235,19 @@ public class BdbClusterSettingEntity implements Serializable {
this.attributes = attributes;
}
- public String getModifyUser() {
- return modifyUser;
+ public void setModifyInfo(String modifyUser, Date modifyDate) {
+ this.modifyUser = modifyUser;
+ this.modifyDate = modifyDate;
}
- public void setModifyUser(String modifyUser) {
- this.modifyUser = modifyUser;
+ public String getModifyUser() {
+ return modifyUser;
}
public Date getModifyDate() {
return modifyDate;
}
- public void setModifyDate(Date modifyDate) {
- this.modifyDate = modifyDate;
- }
-
/**
* Serialize field to json format
*
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
index 86a6bc4..3eae7dc 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
@@ -2020,64 +2020,27 @@ public class BrokerConfManager implements Server {
// /////////////////////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Add cluster default setting
+ * Add or update cluster default setting
*
* @param bdbEntity the cluster default setting entity will be add
* @return true if success otherwise false
* @throws Exception
*/
- public boolean confAddBdbClusterDefSetting(BdbClusterSettingEntity bdbEntity)
+ public boolean confSetBdbClusterDefSetting(BdbClusterSettingEntity bdbEntity)
throws Exception {
validMasterStatus();
- BdbClusterSettingEntity curEntity =
- clusterSettingMap.get(bdbEntity.getRecordKey());
- if (curEntity != null) {
- throw new Exception(new StringBuilder(512)
- .append("Duplicate add ClusterSetting info, exist record is: ")
- .append(curEntity).toString());
- }
- boolean putResult =
+ bdbEntity.setRecordKey(TServerConstants.TOKEN_DEFAULT_CLUSTER_SETTING);
+ boolean result =
mBdbStoreManagerService.putBdbClusterConfEntity(bdbEntity, true);
- if (putResult) {
- clusterSettingMap.put(bdbEntity.getRecordKey(), bdbEntity);
- logger.info(new StringBuilder(512)
- .append("[ClusterSetting Success] ")
- .append(bdbEntity).toString());
- return true;
- }
- return false;
- }
-
- /**
- * update cluster default setting
- *
- * @param bdbEntity the cluster setting entity will be set
- * @return true if success otherwise false
- * @throws Exception
- */
- public boolean confUpdBdbClusterSetting(BdbClusterSettingEntity bdbEntity)
- throws Exception {
- validMasterStatus();
+ clusterSettingMap.put(TServerConstants.TOKEN_DEFAULT_CLUSTER_SETTING, bdbEntity);
StringBuilder strBuffer = new StringBuilder(512);
- BdbClusterSettingEntity curDefSettingEntity =
- clusterSettingMap.get(bdbEntity.getRecordKey());
- if (curDefSettingEntity == null) {
- throw new Exception(strBuffer
- .append("Update ClusterSetting failure, not exist record for record: ")
- .append(bdbEntity.getRecordKey()).toString());
- }
- boolean putResult =
- mBdbStoreManagerService.putBdbClusterConfEntity(bdbEntity, false);
- if (putResult) {
- clusterSettingMap.put(bdbEntity.getRecordKey(), bdbEntity);
- strBuffer.append("[confUpdBdbClusterSetting Success] record from : ");
- strBuffer = curDefSettingEntity.toJsonString(strBuffer);
- strBuffer.append(" to : ");
- strBuffer = bdbEntity.toJsonString(strBuffer);
- logger.info(strBuffer.toString());
- return true;
+ if (result) {
+ strBuffer.append("[confSetBdbClusterDefSetting Success], add new record :");
+ } else {
+ strBuffer.append("[confSetBdbClusterDefSetting Success], update old record :");
}
- return false;
+ logger.info(bdbEntity.toJsonString(strBuffer).toString());
+ return true;
}
/**
@@ -2107,12 +2070,9 @@ public class BrokerConfManager implements Server {
return this.clusterSettingMap.get(TServerConstants.TOKEN_DEFAULT_CLUSTER_SETTING);
}
-
private void validMasterStatus() throws Exception {
if (!isSelfMaster()) {
throw new StandbyException("Please send your request to the master Node.");
}
}
-
-
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
index 98cd36c..9afe1aa 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebMasterInfoHandler.java
@@ -17,10 +17,16 @@
package org.apache.tubemq.server.master.web.handler;
+import java.util.Date;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.utils.SettingValidUtils;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
+import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.common.utils.WebParameterUtils;
import org.apache.tubemq.server.master.TMaster;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbClusterSettingEntity;
import org.apache.tubemq.server.master.web.model.ClusterGroupVO;
import org.apache.tubemq.server.master.web.model.ClusterNodeVO;
@@ -43,9 +49,14 @@ public class WebMasterInfoHandler extends AbstractWebHandler {
// register query method
registerQueryWebMethod("admin_query_master_group_info",
"getGroupAddressStrInfo");
+ registerQueryWebMethod("admin_query_cluster_default_setting",
+ "adminQueryClusterDefSetting");
// register modify method
registerModifyWebMethod("admin_transfer_current_master",
"transferCurrentMaster");
+ // register modify method
+ registerModifyWebMethod("admin_set_cluster_default_setting",
+ "adminSetClusterDefSetting");
}
/**
@@ -94,17 +105,111 @@ public class WebMasterInfoHandler extends AbstractWebHandler {
* @return
*/
public StringBuilder transferCurrentMaster(HttpServletRequest req) {
- StringBuilder strBuffer = new StringBuilder(512);
+ ProcessResult result = new ProcessResult();
+ StringBuilder sBuilder = new StringBuilder(512);
+ // valid operation authorize info
+ if (!WebParameterUtils.validReqAuthorizeInfo(req,
+ WebFieldDef.ADMINAUTHTOKEN, true, master, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
try {
- WebParameterUtils.reqAuthorizeCheck(master, brokerConfManager, req.getParameter("confModAuthToken"));
brokerConfManager.transferMaster();
- strBuffer.append("{\"result\":true,\"errCode\":0," +
- "\"errMsg\":\"TransferMaster method called, please wait 20 seconds!\"}");
+ WebParameterUtils.buildSuccessResult(sBuilder,
+ "TransferMaster method called, please wait 20 seconds!");
} catch (Exception e2) {
- strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
- .append(e2.getMessage()).append("\"}");
+ WebParameterUtils.buildFailResult(sBuilder, e2.getMessage());
}
- return strBuffer;
+ return sBuilder;
+ }
+
+ /**
+ * Query cluster default setting
+ *
+ * @param req
+ * @return
+ * @throws Exception
+ */
+ public StringBuilder adminQueryClusterDefSetting(HttpServletRequest req) {
+ StringBuilder sBuilder = new StringBuilder(512);
+ BdbClusterSettingEntity defClusterSetting =
+ brokerConfManager.getBdbClusterSetting();
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Ok\",\"data\":[");
+ if (defClusterSetting != null) {
+ defClusterSetting.toJsonString(sBuilder);
+ }
+ sBuilder.append("]}");
+ return sBuilder;
+ }
+
+ /**
+ * Add or modify cluster default setting
+ *
+ * @param req
+ * @return
+ */
+ public StringBuilder adminSetClusterDefSetting(HttpServletRequest req) {
+ boolean dataChanged = false;
+ ProcessResult result = new ProcessResult();
+ StringBuilder sBuilder = new StringBuilder(512);
+ // valid operation authorize info
+ if (!WebParameterUtils.validReqAuthorizeInfo(req,
+ WebFieldDef.ADMINAUTHTOKEN, true, master, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
+ // check modify user field
+ if (!WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null, result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
+ String modifyUser = (String) result.retData1;
+ // check max message size
+ if (!WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.MAXMSGSIZE, false,
+ TBaseConstants.META_VALUE_UNDEFINED,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
+ result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
+ int maxMsgSize = (int) result.retData1;
+ if (maxMsgSize != TBaseConstants.META_VALUE_UNDEFINED) {
+ dataChanged = true;
+ }
+ // check and get modify date
+ if (!WebParameterUtils.getDateParameter(req,
+ WebFieldDef.MODIFYDATE, false, new Date(), result)) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return sBuilder;
+ }
+ Date modifyDate = (Date) result.retData1;
+ if (!dataChanged) {
+ WebParameterUtils.buildSuccessResult(sBuilder, "No data is changed!");
+ return sBuilder;
+ }
+ // add or modify cluster setting info
+ BdbClusterSettingEntity defClusterSetting =
+ brokerConfManager.getBdbClusterSetting();
+ if (defClusterSetting == null) {
+ defClusterSetting = new BdbClusterSettingEntity();
+ }
+ defClusterSetting.setModifyInfo(modifyUser, modifyDate);
+ if (maxMsgSize != TBaseConstants.META_VALUE_UNDEFINED) {
+ defClusterSetting.setMaxMsgSize(
+ SettingValidUtils.validAndGetMaxMsgSize(maxMsgSize));
+ }
+ try {
+ brokerConfManager.confSetBdbClusterDefSetting(defClusterSetting);
+ WebParameterUtils.buildSuccessResult(sBuilder);
+ } catch (Exception e) {
+ WebParameterUtils.buildFailResult(sBuilder, e.getMessage());
+ }
+ return sBuilder;
}
+
+
+
}
[incubator-tubemq] 44/49: [TUBEMQ-526] Adjust the packaging script
and version check list, remove the "-WIP" tag
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 2b34005ed3d2698aee60a3829d4182272d573ab5
Author: gosonzhang <go...@tencent.com>
AuthorDate: Tue Jan 19 10:52:24 2021 +0800
[TUBEMQ-526] Adjust the packaging script and version check list, remove the "-WIP" tag
---
pom.xml | 2 +-
tubemq-client/src/main/assembly/assembly.xml | 2 +-
tubemq-example/src/main/assembly/assembly.xml | 2 +-
tubemq-server/src/main/assembly/assembly.xml | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/pom.xml b/pom.xml
index f9b7342..bd51de2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -263,7 +263,7 @@
<!-- Front-end -->
<exclude>resources/assets/lib/**</exclude>
<exclude>resources/assets/public/**</exclude>
- <exclude>DISCLAIMER-WIP</exclude>
+ <exclude>DISCLAIMER</exclude>
<exclude>tubemq-web/**</exclude>
<exclude>**/tubemq-client-twins/tubemq-client-cpp/third_party/**</exclude>
diff --git a/tubemq-client/src/main/assembly/assembly.xml b/tubemq-client/src/main/assembly/assembly.xml
index ee48ac7..52cad36 100644
--- a/tubemq-client/src/main/assembly/assembly.xml
+++ b/tubemq-client/src/main/assembly/assembly.xml
@@ -34,7 +34,7 @@
<include>./conf/tools.log4j.properties</include>
<include>LICENSE</include>
<include>NOTICE</include>
- <include>DISCLAIMER-WIP</include>
+ <include>DISCLAIMER</include>
</includes>
<excludes>
<exclude>**/src/**</exclude>
diff --git a/tubemq-example/src/main/assembly/assembly.xml b/tubemq-example/src/main/assembly/assembly.xml
index 11edd32..34e53e3 100644
--- a/tubemq-example/src/main/assembly/assembly.xml
+++ b/tubemq-example/src/main/assembly/assembly.xml
@@ -34,7 +34,7 @@
<include>./conf/tools.log4j.properties</include>
<include>LICENSE</include>
<include>NOTICE</include>
- <include>DISCLAIMER-WIP</include>
+ <include>DISCLAIMER</include>
</includes>
<excludes>
<exclude>**/src/**</exclude>
diff --git a/tubemq-server/src/main/assembly/assembly.xml b/tubemq-server/src/main/assembly/assembly.xml
index ceb3012..d640901 100644
--- a/tubemq-server/src/main/assembly/assembly.xml
+++ b/tubemq-server/src/main/assembly/assembly.xml
@@ -37,7 +37,7 @@
<include>./resources/**</include>
<include>LICENSE</include>
<include>NOTICE</include>
- <include>DISCLAIMER-WIP</include>
+ <include>DISCLAIMER</include>
</includes>
<excludes>
<exclude>**/src/**</exclude>
[incubator-tubemq] 02/49: [TUBEMQ-440] Add feature package
tube-manager to zip (#337)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit df95aac3aec661c43e502084b02713bad1d5f182
Author: EMsnap <zp...@connect.ust.hk>
AuthorDate: Thu Dec 3 17:40:37 2020 -0800
[TUBEMQ-440] Add feature package tube-manager to zip (#337)
---
tubemq-manager/conf/logback.xml | 64 +++++++++++++++++++++++++++
tubemq-manager/pom.xml | 31 ++++++++++++-
tubemq-manager/src/main/assembly/assembly.xml | 61 +++++++++++++++++++++++++
3 files changed, 154 insertions(+), 2 deletions(-)
diff --git a/tubemq-manager/conf/logback.xml b/tubemq-manager/conf/logback.xml
new file mode 100644
index 0000000..3e4259f
--- /dev/null
+++ b/tubemq-manager/conf/logback.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<configuration>
+ <property name="FILE_ERROR_PATTERN"
+ value="${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} %file:%line: %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
+ <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <filter class="ch.qos.logback.classic.filter.LevelFilter">
+ <level>INFO</level>
+ </filter>
+ <encoder>
+ <pattern>${CONSOLE_LOG_PATTERN}</pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <appender name="FILE_INFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <FileNamePattern>../logs/tubemq-manager.info.%d{yyyy-MM-dd}.part_%i.log</FileNamePattern>
+ <maxHistory>10</maxHistory>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>100MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ </rollingPolicy>
+ <encoder>
+ <pattern>${FILE_LOG_PATTERN}</pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <appender name="FILE_ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <FileNamePattern>../logs/tubemq-manager.error.%d{yyyy-MM-dd}.part_%i.log</FileNamePattern>
+ <maxHistory>10</maxHistory>
+ <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+ <maxFileSize>100MB</maxFileSize>
+ </timeBasedFileNamingAndTriggeringPolicy>
+ </rollingPolicy>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>Error</level>
+ </filter>
+ <encoder>
+ <pattern>${FILE_ERROR_PATTERN}</pattern>
+ <charset>UTF-8</charset>
+ </encoder>
+ </appender>
+
+ <root level="info">
+ <appender-ref ref="FILE_INFO"/>
+ <appender-ref ref="FILE_ERROR"/>
+ </root>
+</configuration>
diff --git a/tubemq-manager/pom.xml b/tubemq-manager/pom.xml
index b5bf2d7..1fa0994 100644
--- a/tubemq-manager/pom.xml
+++ b/tubemq-manager/pom.xml
@@ -111,8 +111,35 @@
<build>
<plugins>
<plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.3.0</version>
+ <configuration>
+ <descriptors>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </descriptors>
+ <archive>
+ <manifest>
+ <mainClass>com.cyy.MvnPackageApplication</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}</finalName>
+ <descriptors>
+ <descriptor>src/main/assembly/assembly.xml</descriptor>
+ </descriptors>
+ <outputDirectory>./dist/</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
diff --git a/tubemq-manager/src/main/assembly/assembly.xml b/tubemq-manager/src/main/assembly/assembly.xml
new file mode 100644
index 0000000..82a8b8a
--- /dev/null
+++ b/tubemq-manager/src/main/assembly/assembly.xml
@@ -0,0 +1,61 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
+ <id>bin</id>
+ <formats>
+ <format>zip</format>
+ </formats>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ </dependencySet>
+ </dependencySets>
+ <fileSets>
+ <fileSet>
+ <directory>bin</directory>
+ <outputDirectory>/bin</outputDirectory>
+ <includes>
+ <include>*.sh</include>
+ <include>*.bat</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>conf</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ <include>*.properties</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources</directory>
+ <outputDirectory>/conf</outputDirectory>
+ <includes>
+ <include>*.xml</include>
+ <include>*.properties</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <includes>
+ <include>tubemq-manager-*.jar</include>
+ </includes>
+ <outputDirectory>/lib</outputDirectory>
+ </fileSet>
+
+ </fileSets>
+</assembly>
\ No newline at end of file
[incubator-tubemq] 49/49: [TUBEMQ-547]Recode the implementation of
the *Startup.java classes in the Tool package
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit e69631b0862220463cdc7bd6008ed7eac9c00284
Author: gosonzhang <go...@tencent.com>
AuthorDate: Thu Jan 28 17:42:35 2021 +0800
[TUBEMQ-547]Recode the implementation of the *Startup.java classes in the Tool package
---
LICENSE | 1 -
.../tubemq/server/common/fielddef/CliArgDef.java | 6 +-
.../apache/tubemq/server/tools/BrokerStartup.java | 24 ++++++-
.../org/apache/tubemq/server/tools/CliUtils.java | 72 ++++++++++++++++++++
.../apache/tubemq/server/tools/MasterStartup.java | 21 +++++-
.../org/apache/tubemq/server/tools/ToolUtils.java | 78 ----------------------
6 files changed, 116 insertions(+), 86 deletions(-)
diff --git a/LICENSE b/LICENSE
index 4834d76..a2d97e6 100644
--- a/LICENSE
+++ b/LICENSE
@@ -303,7 +303,6 @@ License: https://github.com/webx/citrus/blob/r3.1.4/pom.xml
5. bin/broker.sh
bin/master.sh
- tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
Metamorphosis metamorphosis-all-1.4.4 (Please note that the software have been modified.)
Copyrighy (C) Alibaba Group Holding Limited.
License: https://github.com/killme2008/Metamorphosis/blob/metamorphosis-all-1.4.4/COPYING.txt
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
index 2e39f2f..1e1feea 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
@@ -100,8 +100,10 @@ public enum CliArgDef {
"String: http call method",
"Http call method"),
ADMINMETHOD(null, "show-methods",
- "Return http's methods.");
-
+ "Return http's methods."),
+ FILEPATH("f", "file",
+ "String: file path.",
+ "File path.");
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/BrokerStartup.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/BrokerStartup.java
index a966278..12d6189 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/BrokerStartup.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/BrokerStartup.java
@@ -19,12 +19,30 @@ package org.apache.tubemq.server.tools;
import org.apache.tubemq.server.broker.BrokerConfig;
import org.apache.tubemq.server.broker.TubeBroker;
+import org.apache.tubemq.server.common.utils.ProcessResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class BrokerStartup {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(BrokerStartup.class);
+
public static void main(final String[] args) throws Exception {
- final String configFilePath = ToolUtils.getConfigFilePath(args);
- final BrokerConfig tubeConfig = ToolUtils.getBrokerConfig(configFilePath);
- final TubeBroker server = new TubeBroker(tubeConfig);
+ // get configure file path
+ ProcessResult result = new ProcessResult();
+ if (!CliUtils.getConfigFilePath(args, result)) {
+ System.err.println(result.errInfo);
+ System.exit(1);
+ }
+ String configFilePath = (String) result.retData1;
+ // read configure file
+ BrokerConfig brokerConfig = new BrokerConfig();
+ brokerConfig.loadFromFile(configFilePath);
+ logger.info("[BrokerStartup] Broker config is: " + brokerConfig);
+ // start broker instance
+ final TubeBroker server = new TubeBroker(brokerConfig);
server.start();
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/CliUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/CliUtils.java
new file mode 100644
index 0000000..d2a25c5
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/CliUtils.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.tools;
+
+import static org.apache.tubemq.server.common.fielddef.CliArgDef.FILEPATH;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.server.common.utils.ProcessResult;
+
+
+
+public class CliUtils {
+
+ public static boolean getConfigFilePath(final String[] args, ProcessResult result) {
+ // build file option
+ Options options = new Options();
+ Option fileOption = new Option(FILEPATH.opt,
+ FILEPATH.longOpt, FILEPATH.hasArg, FILEPATH.optDesc);
+ if (FILEPATH.hasArg) {
+ fileOption.setArgName(FILEPATH.argDesc);
+ }
+ options.addOption(fileOption);
+ // parse args
+ CommandLineParser parser = new DefaultParser();
+ try {
+ CommandLine cli = parser.parse(options, args);
+ if (cli == null) {
+ result.setFailResult("Parse args failure");
+ return result.success;
+ }
+ if (!cli.hasOption(FILEPATH.longOpt)) {
+ result.setFailResult(new StringBuilder(512)
+ .append("Please input the configuration file path by ")
+ .append("-").append(FILEPATH.opt).append(" or ")
+ .append("-").append(FILEPATH.longOpt).append(" option").toString());
+ return result.success;
+ }
+ String configFilePath = cli.getOptionValue(FILEPATH.longOpt);
+ if (TStringUtils.isBlank(configFilePath)) {
+ result.setFailResult(new StringBuilder(512)
+ .append(FILEPATH.longOpt).append(" is required!").toString());
+ return result.success;
+ }
+ result.setSuccResult(configFilePath);
+ } catch (Throwable e) {
+ result.setFailResult(new StringBuilder(512)
+ .append("Parse configuration file path failure: ")
+ .append(e.toString()).toString());
+ }
+ return result.success;
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/MasterStartup.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/MasterStartup.java
index a770f40..1d441f1 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/MasterStartup.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/MasterStartup.java
@@ -17,13 +17,30 @@
package org.apache.tubemq.server.tools;
+import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.master.MasterConfig;
import org.apache.tubemq.server.master.TMaster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MasterStartup {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(MasterStartup.class);
+
public static void main(final String[] args) throws Exception {
- final String configFilePath = ToolUtils.getConfigFilePath(args);
- final MasterConfig masterConfig = ToolUtils.getMasterConfig(configFilePath);
+ // get configure file path
+ ProcessResult result = new ProcessResult();
+ if (!CliUtils.getConfigFilePath(args, result)) {
+ System.err.println(result.errInfo);
+ System.exit(1);
+ }
+ String configFilePath = (String) result.retData1;
+ // read configure file
+ final MasterConfig masterConfig = new MasterConfig();
+ masterConfig.loadFromFile(configFilePath);
+ logger.info("[MasterStartup] master config is: " + masterConfig);
+ // start master instance
TMaster master = new TMaster(masterConfig);
master.start();
master.join();
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
deleted file mode 100644
index 9c343a8..0000000
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tubemq.server.tools;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.tubemq.corebase.utils.TStringUtils;
-import org.apache.tubemq.server.broker.BrokerConfig;
-import org.apache.tubemq.server.broker.exception.StartupException;
-import org.apache.tubemq.server.master.MasterConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Utility for .ini configure file loading.
- * Copied from <a href="https://github.com/killme2008/Metamorphosis">Metamorphosis Project</a>
- */
-public class ToolUtils {
-
- private static final Logger logger = LoggerFactory.getLogger(ToolUtils.class);
-
- public static BrokerConfig getBrokerConfig(final String configFilePath) {
- final BrokerConfig brokerConfig = new BrokerConfig();
- brokerConfig.loadFromFile(configFilePath);
- logger.info("Broker config is: " + brokerConfig);
- return brokerConfig;
- }
-
- public static MasterConfig getMasterConfig(final String configFilePath) {
- final MasterConfig masterConfig = new MasterConfig();
- masterConfig.loadFromFile(configFilePath);
- logger.info("master config is: " + masterConfig);
- return masterConfig;
- }
-
- public static String getConfigFilePath(final String[] args) throws StartupException {
- final Options options = new Options();
- final Option file = new Option("f", true, "configuration file path");
- options.addOption(file);
- final CommandLineParser parser = new DefaultParser();
- CommandLine line = null;
- try {
- line = parser.parse(options, args);
- } catch (final ParseException e) {
- throw new StartupException("Parse command line failed", e);
- }
- String configFilePath = null;
- if (line.hasOption("f")) {
- configFilePath = line.getOptionValue("f");
- } else {
- System.err.println("Please tell me the configuration file path by -f option");
- System.exit(1);
- }
- if (TStringUtils.isBlank(configFilePath)) {
- throw new StartupException("Blank file path");
- }
- return configFilePath;
- }
-}
[incubator-tubemq] 11/49: [TUBEMQ-446]Small bugs fix that do not
affect the main logics (#346)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 40a34916c63be0a8b0fae3e0525d0671019f88d7
Author: gosonzhang <46...@qq.com>
AuthorDate: Thu Dec 10 09:24:52 2020 +0800
[TUBEMQ-446]Small bugs fix that do not affect the main logics (#346)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java | 3 +++
.../main/java/org/apache/tubemq/example/MessageConsumerExample.java | 2 +-
.../src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java | 1 -
3 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
index 2828bf7..206cb93 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
@@ -144,6 +144,9 @@ public class AddressUtils {
}
public static String getIPV4LocalAddress() {
+ if (localIPAddress != null) {
+ return localIPAddress;
+ }
String tmpAdress = null;
try {
Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
index d9aeb8a..0252d16 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
@@ -96,7 +96,7 @@ public final class MessageConsumerExample {
topicTidsMap.put(topicTidStr[0], tids);
}
final int startFetchCount = fetchCount;
- final ExecutorService executorService = Executors.newFixedThreadPool(fetchCount);
+ final ExecutorService executorService = Executors.newCachedThreadPool();
for (int i = 0; i < consumerCount; i++) {
executorService.submit(new Runnable() {
@Override
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
index 666e6f9..359ea44 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
@@ -103,7 +103,6 @@ public class CliConsumer extends CliAbstractBase {
addCommandOption(CliArgDef.FETCHTHREADS);
addCommandOption(CliArgDef.CLIENTCOUNT);
addCommandOption(CliArgDef.OUTPUTINTERVAL);
- addCommandOption(CliArgDef.WITHOUTDELAY);
}
public boolean parseParams(String[] args) throws Exception {
[incubator-tubemq] 39/49: [TUBEMQ-511]Replace the conditional
operator (?:) with mid()
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 78e368526c9f62e84c50ea77665ba8ba15d1515a
Author: gosonzhang <go...@tencent.com>
AuthorDate: Wed Jan 13 19:51:52 2021 +0800
[TUBEMQ-511]Replace the conditional operator (?:) with mid()
---
.../apache/tubemq/server/broker/msgstore/MessageStore.java | 13 +++++--------
.../tubemq/server/broker/offset/DefaultOffsetManager.java | 10 ++++------
.../apache/tubemq/server/broker/web/BrokerAdminServlet.java | 13 +++++--------
.../java/org/apache/tubemq/server/master/MasterConfig.java | 3 ++-
.../org/apache/tubemq/server/tools/cli/CliConsumer.java | 2 +-
.../org/apache/tubemq/server/tools/cli/CliProducer.java | 2 +-
6 files changed, 18 insertions(+), 25 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
index 8f650b4..1987c32 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
@@ -35,6 +35,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TErrCodeConstants;
import org.apache.tubemq.corebase.protobuf.generated.ClientBroker;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.ThreadUtils;
import org.apache.tubemq.server.broker.BrokerConfig;
import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
@@ -133,10 +134,8 @@ public class MessageStore implements Closeable {
this.writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata.getMemCacheMsgSize());
this.writeCacheFlushIntvl = topicMetadata.getMemCacheFlushIntvl();
int tmpIndexReadCnt = tubeConfig.getIndexTransCount() * partitionNum;
- memMaxIndexReadCnt.set(tmpIndexReadCnt <= 6000
- ? 6000 : (Math.min(tmpIndexReadCnt, 10000)));
- fileMaxIndexReadCnt.set(tmpIndexReadCnt < 8000
- ? 8000 : (Math.min(tmpIndexReadCnt, 13500)));
+ memMaxIndexReadCnt.set(MixedUtils.mid(tmpIndexReadCnt, 6000, 10000));
+ fileMaxIndexReadCnt.set(MixedUtils.mid(tmpIndexReadCnt, 8000, 13500));
memMaxFilterIndexReadCnt.set(memMaxIndexReadCnt.get() * 2);
fileMaxFilterIndexReadCnt.set(fileMaxIndexReadCnt.get() * 3);
fileLowReqMaxFilterIndexReadCnt.set(fileMaxFilterIndexReadCnt.get() * 10);
@@ -408,10 +407,8 @@ public class MessageStore implements Closeable {
unflushDataHold.set(topicMetadata.getUnflushDataHold());
maxFileValidDurMs.set(parseDeletePolicy(topicMetadata.getDeletePolicy()));
int tmpIndexReadCnt = tubeConfig.getIndexTransCount() * partitionNum;
- memMaxIndexReadCnt.set(tmpIndexReadCnt <= 6000
- ? 6000 : (Math.min(tmpIndexReadCnt, 10000)));
- fileMaxIndexReadCnt.set(tmpIndexReadCnt < 8000
- ? 8000 : (Math.min(tmpIndexReadCnt, 13500)));
+ memMaxIndexReadCnt.set(MixedUtils.mid(tmpIndexReadCnt, 6000, 10000));
+ fileMaxIndexReadCnt.set(MixedUtils.mid(tmpIndexReadCnt, 8000, 13500));
memMaxFilterIndexReadCnt.set(memMaxIndexReadCnt.get() * 2);
fileMaxFilterIndexReadCnt.set(fileMaxIndexReadCnt.get() * 3);
fileLowReqMaxFilterIndexReadCnt.set(fileMaxFilterIndexReadCnt.get() * 10);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
index 0e91dc2..50c8bd3 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.daemon.AbstractDaemonService;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.corebase.utils.Tuple3;
@@ -121,8 +122,7 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
|| (readStatus == TBaseConstants.CONSUME_MODEL_READ_FROM_MAX_ALWAYS)) {
long adjOffset = indexMaxOffset;
if (readStatus != TBaseConstants.CONSUME_MODEL_READ_FROM_MAX_ALWAYS) {
- adjOffset = Math.min(reqOffset, indexMaxOffset);
- adjOffset = Math.max(adjOffset, indexMinOffset);
+ adjOffset = MixedUtils.mid(reqOffset, indexMinOffset, indexMaxOffset);
}
regInfo.getAndSetOffset(adjOffset);
}
@@ -287,10 +287,8 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
long reSetOffset, final String modifier) {
long oldOffset = -1;
if (store != null) {
- long firstOffset = store.getIndexMinOffset();
- long lastOffset = store.getIndexMaxOffset();
- reSetOffset = reSetOffset < firstOffset
- ? firstOffset : Math.min(reSetOffset, lastOffset);
+ reSetOffset = MixedUtils.mid(reSetOffset,
+ store.getIndexMinOffset(), store.getIndexMaxOffset());
String offsetCacheKey = getOffsetCacheKey(topic, partitionId);
getAndResetTmpOffset(group, offsetCacheKey);
OffsetStorageInfo regInfo =
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index d35ba91..dd4bbdf 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TokenConstants;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.corebase.utils.Tuple3;
@@ -959,11 +960,9 @@ public class BrokerAdminServlet extends AbstractWebHandler {
if (store == null) {
continue;
}
- long firstOffset = store.getIndexMinOffset();
- long lastOffset = store.getIndexMaxOffset();
// adjust reset offset value
- adjOffset = offsetTuple.getF0() < firstOffset
- ? firstOffset : Math.min(offsetTuple.getF0(), lastOffset);
+ adjOffset = MixedUtils.mid(offsetTuple.getF0(),
+ store.getIndexMinOffset(), store.getIndexMaxOffset());
result.add(new Tuple3<>(entry.getKey(), entry1.getKey(), adjOffset));
}
}
@@ -1059,10 +1058,8 @@ public class BrokerAdminServlet extends AbstractWebHandler {
if (store == null) {
continue;
}
- long firstOffset = store.getIndexMinOffset();
- long lastOffset = store.getIndexMaxOffset();
- adjOffset = entry.getValue() < firstOffset
- ? firstOffset : Math.min(entry.getValue(), lastOffset);
+ adjOffset = MixedUtils.mid(entry.getValue(),
+ store.getIndexMinOffset(), store.getIndexMaxOffset());
offsetVals.add(new Tuple3<>(topicName, partitionId, adjOffset));
}
if (offsetVals.isEmpty()) {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java
index b112d66..c2a8007 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java
@@ -22,6 +22,7 @@ import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.config.TLSConfig;
import org.apache.tubemq.corebase.utils.AddressUtils;
+import org.apache.tubemq.corebase.utils.MixedUtils;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corerpc.RpcConstants;
import org.apache.tubemq.server.common.TServerConstants;
@@ -467,7 +468,7 @@ public class MasterConfig extends AbstractFileConfig {
}
if (TStringUtils.isNotBlank(masterConf.get("rebalanceParallel"))) {
int tmpParallel = this.getInt(masterConf, "rebalanceParallel");
- this.rebalanceParallel = (tmpParallel <= 0) ? 1 : (Math.min(tmpParallel, 20));
+ this.rebalanceParallel = MixedUtils.mid(tmpParallel, 1, 20);
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
index c8938a6..52f0800 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
@@ -177,7 +177,7 @@ public class CliConsumer extends CliAbstractBase {
String fetchThreadCntStr = cli.getOptionValue(CliArgDef.FETCHTHREADS.longOpt);
if (TStringUtils.isNotBlank(fetchThreadCntStr)) {
int tmpFetchThreadCnt = Integer.parseInt(fetchThreadCntStr);
- tmpFetchThreadCnt = (tmpFetchThreadCnt < 1) ? 1 : Math.min(tmpFetchThreadCnt, 100);
+ tmpFetchThreadCnt = MixedUtils.mid(tmpFetchThreadCnt, 1, 100);
fetchThreadCnt = tmpFetchThreadCnt;
}
return true;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
index a6e7e38..8a01e29 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
@@ -152,7 +152,7 @@ public class CliProducer extends CliAbstractBase {
String sendThreadCntStr = cli.getOptionValue(CliArgDef.SENDTHREADS.longOpt);
if (TStringUtils.isNotBlank(sendThreadCntStr)) {
int tmpThreadCnt = Integer.parseInt(sendThreadCntStr);
- tmpThreadCnt = (tmpThreadCnt < 1) ? 1 : Math.min(tmpThreadCnt, 200);
+ tmpThreadCnt = MixedUtils.mid(tmpThreadCnt, 1, 200);
sendThreadCnt = tmpThreadCnt;
}
String rpcTimeoutStr = cli.getOptionValue(CliArgDef.RPCTIMEOUT.longOpt);
[incubator-tubemq] 40/49: [TUBEMQ-512] Add package length control
based on Topic
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 5e4129e07591b1486c823f9377ddf30a984d2bd2
Author: gosonzhang <go...@tencent.com>
AuthorDate: Fri Jan 15 17:38:25 2021 +0800
[TUBEMQ-512] Add package length control based on Topic
---
.../tubemq/server/broker/BrokerServiceServer.java | 12 +++---
.../broker/metadata/ClusterConfigHolder.java | 25 ++++++++-----
.../server/broker/metadata/TopicMetadata.java | 43 +++++++++++++++++++++-
.../server/broker/msgstore/MessageStore.java | 14 ++++---
.../tubemq/server/common/TServerConstants.java | 1 +
.../server/common/paramcheck/PBParameterUtils.java | 2 +-
.../bdbstore/bdbentitys/BdbTopicConfEntity.java | 18 +++++++++
.../nodemanage/nodebroker/BrokerConfManager.java | 6 +++
.../nodebroker/BrokerSyncStatusInfo.java | 15 ++++++++
.../web/handler/WebBrokerTopicConfHandler.java | 43 ++++++++++++++++++++--
10 files changed, 153 insertions(+), 26 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
index c9cfba4..a95fa09 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/BrokerServiceServer.java
@@ -55,8 +55,8 @@ import org.apache.tubemq.corerpc.RpcConstants;
import org.apache.tubemq.corerpc.service.BrokerReadService;
import org.apache.tubemq.corerpc.service.BrokerWriteService;
import org.apache.tubemq.server.Server;
-import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.metadata.MetadataManager;
+import org.apache.tubemq.server.broker.metadata.TopicMetadata;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.msgstore.disk.GetMessageResult;
@@ -621,7 +621,8 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String topicName = (String) result.retData1;
+ final TopicMetadata topicMetadata = (TopicMetadata) result.retData1;
+ final String topicName = topicMetadata.getTopic();
String msgType = null;
int msgTypeCode = -1;
if (TStringUtils.isNotBlank(request.getMsgType())) {
@@ -635,10 +636,10 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg("data length is zero!");
return builder.build();
}
- if (dataLength > ClusterConfigHolder.getMaxMsgSize()) {
+ if (dataLength > topicMetadata.getMaxMsgSize()) {
builder.setErrCode(TErrCodeConstants.BAD_REQUEST);
builder.setErrMsg(strBuffer.append("data length over max length, allowed max length is ")
- .append(ClusterConfigHolder.getMaxMsgSize())
+ .append(topicMetadata.getMaxMsgSize())
.append(", data length is ").append(dataLength).toString());
return builder.build();
}
@@ -1137,7 +1138,8 @@ public class BrokerServiceServer implements BrokerReadService, BrokerWriteServic
builder.setErrMsg(result.errInfo);
return builder.build();
}
- final String topicName = (String) result.retData1;
+ final TopicMetadata topicMetadata = (TopicMetadata) result.retData1;
+ final String topicName = topicMetadata.getTopic();
String partStr = getPartStr(groupName, topicName, partitionId);
ConsumerNodeInfo consumerNodeInfo = consumerRegisterMap.get(partStr);
if (consumerNodeInfo == null) {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
index 1b370d5..7d89dba 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.protobuf.generated.ClientMaster;
import org.apache.tubemq.corebase.utils.MixedUtils;
-
+import org.apache.tubemq.corebase.utils.Tuple2;
public class ClusterConfigHolder {
@@ -46,14 +46,11 @@ public class ClusterConfigHolder {
if (configId.get() != clusterConfig.getConfigId()) {
configId.set(clusterConfig.getConfigId());
if (clusterConfig.hasMaxMsgSize()) {
- int tmpMaxSize = MixedUtils.mid(clusterConfig.getMaxMsgSize(),
- TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
- TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT)
- + TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE;
- if (tmpMaxSize != maxMsgSize.get()) {
- maxMsgSize.set(tmpMaxSize);
- minMemCacheSize.set(tmpMaxSize +
- (tmpMaxSize % 4 + 1) * TBaseConstants.META_MESSAGE_SIZE_ADJUST);
+ Tuple2<Integer, Integer> calcResult =
+ calcMaxMsgSize(clusterConfig.getMaxMsgSize());
+ if (calcResult.getF0() != maxMsgSize.get()) {
+ maxMsgSize.set(calcResult.getF0());
+ minMemCacheSize.set(calcResult.getF1());
}
}
}
@@ -71,4 +68,14 @@ public class ClusterConfigHolder {
return minMemCacheSize.get();
}
+ public static Tuple2<Integer, Integer> calcMaxMsgSize(int maxMsgSize) {
+ int tmpMaxSize = MixedUtils.mid(maxMsgSize,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE,
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT)
+ + TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE;
+ int tmpMinMemCacheSize = tmpMaxSize +
+ (tmpMaxSize % 4 + 1) * TBaseConstants.META_MESSAGE_SIZE_ADJUST;
+ return new Tuple2<>(tmpMaxSize, tmpMinMemCacheSize);
+ }
+
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
index 800254b..6e47d63 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
@@ -25,6 +25,7 @@ import java.util.Set;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.common.TStatusConstants;
@@ -62,6 +63,10 @@ public class TopicMetadata {
private int memCacheMsgCnt = 5 * 1024;
// the max interval(milliseconds) that topic's memory cache will flush to disk.
private int memCacheFlushIntvl = 20000;
+ // the allowed max message size
+ private int maxMsgSize = TBaseConstants.META_VALUE_UNDEFINED;
+ // the allowed min memory cache size
+ private int minMemCacheSize = TBaseConstants.META_VALUE_UNDEFINED;
/***
* Build TopicMetadata from brokerDefMetadata(default config) and topicMetaConfInfo(custom config).
@@ -141,6 +146,17 @@ public class TopicMetadata {
} else {
this.memCacheFlushIntvl = Integer.parseInt(topicConfInfoArr[13]);
}
+ this.maxMsgSize = ClusterConfigHolder.getMaxMsgSize();
+ this.minMemCacheSize = ClusterConfigHolder.getMinMemCacheSize();
+ if (topicConfInfoArr.length > 14) {
+ if (TStringUtils.isNotBlank(topicConfInfoArr[14])) {
+ int maxMsgSize = Integer.parseInt(topicConfInfoArr[14]);
+ Tuple2<Integer, Integer> calcResult =
+ ClusterConfigHolder.calcMaxMsgSize(maxMsgSize);
+ this.maxMsgSize = calcResult.getF0();
+ this.minMemCacheSize = calcResult.getF1();
+ }
+ }
}
private TopicMetadata(String topic, int unflushThreshold,
@@ -149,7 +165,8 @@ public class TopicMetadata {
int numPartitions, boolean acceptPublish,
boolean acceptSubscribe, int statusId,
int numTopicStores, int memCacheMsgSize,
- int memCacheMsgCnt, int memCacheFlushIntvl) {
+ int memCacheMsgCnt, int memCacheFlushIntvl,
+ int maxMsgSize, int minMemCacheSize) {
this.topic = topic;
this.unflushThreshold = unflushThreshold;
this.unflushInterval = unflushInterval;
@@ -165,6 +182,8 @@ public class TopicMetadata {
this.memCacheMsgSize = memCacheMsgSize;
this.memCacheMsgCnt = memCacheMsgCnt;
this.memCacheFlushIntvl = memCacheFlushIntvl;
+ this.maxMsgSize = maxMsgSize;
+ this.minMemCacheSize = minMemCacheSize;
}
@Override
@@ -175,7 +194,8 @@ public class TopicMetadata {
this.numPartitions, this.acceptPublish,
this.acceptSubscribe, this.statusId,
this.numTopicStores, this.memCacheMsgSize,
- this.memCacheMsgCnt, this.memCacheFlushIntvl);
+ this.memCacheMsgCnt, this.memCacheFlushIntvl,
+ this.maxMsgSize, this.minMemCacheSize);
}
public boolean isAcceptPublish() {
@@ -304,6 +324,14 @@ public class TopicMetadata {
return memCacheFlushIntvl;
}
+ public int getMaxMsgSize() {
+ return maxMsgSize;
+ }
+
+ public int getMinMemCacheSize() {
+ return minMemCacheSize;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
@@ -323,6 +351,8 @@ public class TopicMetadata {
result = prime * result + this.memCacheMsgSize;
result = prime * result + this.memCacheMsgCnt;
result = prime * result + this.memCacheFlushIntvl;
+ result = prime * result + this.maxMsgSize;
+ result = prime * result + this.minMemCacheSize;
return result;
}
@@ -400,6 +430,12 @@ public class TopicMetadata {
if (this.memCacheFlushIntvl != other.memCacheFlushIntvl) {
return false;
}
+ if (this.maxMsgSize != other.maxMsgSize) {
+ return false;
+ }
+ if (this.minMemCacheSize != other.minMemCacheSize) {
+ return false;
+ }
return true;
}
@@ -418,6 +454,7 @@ public class TopicMetadata {
&& this.memCacheMsgSize == other.memCacheMsgSize
&& this.memCacheMsgCnt == other.memCacheMsgCnt
&& this.memCacheFlushIntvl == other.memCacheFlushIntvl
+ && this.maxMsgSize == other.maxMsgSize
&& this.deletePolicy.equals(other.deletePolicy));
}
@@ -438,6 +475,8 @@ public class TopicMetadata {
.append(", memCacheMsgSizeInMs=").append(this.memCacheMsgSize / 1024 / 512)
.append(", memCacheMsgCntInK=").append(this.memCacheMsgCnt / 512)
.append(", memCacheFlushIntvl=").append(this.memCacheFlushIntvl)
+ .append(", maxMsgSize=").append(this.maxMsgSize)
+ .append(", minMemCacheSize=").append(this.minMemCacheSize)
.append("]").toString();
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
index 1987c32..a025f7c 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
@@ -131,7 +131,7 @@ public class MessageStore implements Closeable {
this.unflushThreshold.set(topicMetadata.getUnflushThreshold());
this.unflushDataHold.set(topicMetadata.getUnflushDataHold());
this.writeCacheMaxCnt = topicMetadata.getMemCacheMsgCnt();
- this.writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata.getMemCacheMsgSize());
+ this.writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata);
this.writeCacheFlushIntvl = topicMetadata.getMemCacheFlushIntvl();
int tmpIndexReadCnt = tubeConfig.getIndexTransCount() * partitionNum;
memMaxIndexReadCnt.set(MixedUtils.mid(tmpIndexReadCnt, 6000, 10000));
@@ -419,7 +419,7 @@ public class MessageStore implements Closeable {
writeCacheMutex.readLock().lock();
try {
writeCacheMaxCnt = topicMetadata.getMemCacheMsgCnt();
- writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata.getMemCacheMsgSize());
+ writeCacheMaxSize = validAndGetMemCacheSize(topicMetadata);
writeCacheFlushIntvl = topicMetadata.getMemCacheFlushIntvl();
} finally {
writeCacheMutex.readLock().unlock();
@@ -601,13 +601,15 @@ public class MessageStore implements Closeable {
}
}
- private int validAndGetMemCacheSize(int memCacheSize) {
- if (memCacheSize <= ClusterConfigHolder.getMinMemCacheSize()) {
+ private int validAndGetMemCacheSize(TopicMetadata topicMetadata) {
+ int memCacheSize = topicMetadata.getMemCacheMsgSize();
+ if (memCacheSize <= topicMetadata.getMinMemCacheSize()) {
logger.info(new StringBuilder(512)
- .append("[Data Store] writeCacheMaxSize changed, from ")
+ .append("[Data Store] ").append(getTopic())
+ .append(" writeCacheMaxSize changed, from ")
.append(memCacheSize).append(" to ")
.append(ClusterConfigHolder.getMinMemCacheSize()).toString());
- memCacheSize = ClusterConfigHolder.getMinMemCacheSize();
+ memCacheSize = topicMetadata.getMinMemCacheSize();
}
return memCacheSize;
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java
index 5793364..4e2547a 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java
@@ -26,6 +26,7 @@ public final class TServerConstants {
public static final String TOKEN_JOB_STORE_MGR = "messageStoreManager";
public static final String TOKEN_DEFAULT_FLOW_CONTROL = "default_master_ctrl";
public static final String TOKEN_DEFAULT_CLUSTER_SETTING = "default_cluster_config";
+ public static final String TOKEN_MAX_MSG_SIZE = "maxMsgSize";
public static final String TOKEN_BLANK_FILTER_CONDITION = ",,";
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
index 1c2d5ca..c27ad5b 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/paramcheck/PBParameterUtils.java
@@ -608,7 +608,7 @@ public class PBParameterUtils {
strBuffer.delete(0, strBuffer.length());
return result.success;
}
- result.setSuccResult(tmpValue);
+ result.setSuccResult(topicMetadata);
return result.success;
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbTopicConfEntity.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbTopicConfEntity.java
index 3763ef9..de9f6ef 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbTopicConfEntity.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbTopicConfEntity.java
@@ -345,6 +345,23 @@ public class BdbTopicConfEntity implements Serializable {
String.valueOf(memCacheFlushIntvl));
}
+ public int getMaxMsgSize() {
+ String atrVal =
+ TStringUtils.getAttrValFrmAttributes(this.attributes,
+ TServerConstants.TOKEN_MAX_MSG_SIZE);
+ if (atrVal != null) {
+ return Integer.parseInt(atrVal);
+ }
+ return TBaseConstants.META_VALUE_UNDEFINED;
+ }
+
+ public void setMaxMsgSize(int maxMsgSize) {
+ this.attributes =
+ TStringUtils.setAttrValToAttributes(this.attributes,
+ TServerConstants.TOKEN_MAX_MSG_SIZE,
+ String.valueOf(maxMsgSize));
+ }
+
public void appendAttributes(String attrKey, String attrVal) {
this.attributes =
TStringUtils.setAttrValToAttributes(this.attributes, attrKey, attrVal);
@@ -375,6 +392,7 @@ public class BdbTopicConfEntity implements Serializable {
.append(",\"memCacheMsgCntInK\":").append(getMemCacheMsgCntInK())
.append(",\"memCacheMsgSizeInMB\":").append(getMemCacheMsgSizeInMB())
.append(",\"memCacheFlushIntvl\":").append(getMemCacheFlushIntvl())
+ .append(",\"maxMsgSize\":").append(getMaxMsgSize())
.append(",\"dataPath\":\"").append(dataPath)
.append("\",\"createUser\":\"").append(createUser)
.append("\",\"createDate\":\"")
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
index 3eae7dc..56f1914 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
@@ -1205,6 +1205,12 @@ public class BrokerConfManager implements Server {
} else {
sbuffer.append(TokenConstants.ATTR_SEP).append(topicEntity.getMemCacheFlushIntvl());
}
+ int maxMsgSize = topicEntity.getMaxMsgSize();
+ if (maxMsgSize == TBaseConstants.META_VALUE_UNDEFINED) {
+ sbuffer.append(TokenConstants.ATTR_SEP).append(" ");
+ } else {
+ sbuffer.append(TokenConstants.ATTR_SEP).append(maxMsgSize);
+ }
brokerTopicStrConfSet.add(sbuffer.toString());
sbuffer.delete(0, sbuffer.length());
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerSyncStatusInfo.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerSyncStatusInfo.java
index 176c39a..83e2794 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerSyncStatusInfo.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerSyncStatusInfo.java
@@ -26,6 +26,8 @@ import org.apache.commons.codec.binary.StringUtils;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.CheckSum;
import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
+import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.TStatusConstants;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
@@ -730,6 +732,8 @@ public class BrokerSyncStatusInfo {
int tmpmemCacheMsgSizeInMB = memCacheMsgSizeInMB;
int tmpmemCacheMsgCntInK = memCacheMsgCntInK;
int tmpmemCacheFlushIntvl = memCacheFlushIntvl;
+ int tmpMaxMsgSize = ClusterConfigHolder.getMaxMsgSize();
+ int tmpMinMemCacheSize = ClusterConfigHolder.getMinMemCacheSize();
if (!TStringUtils.isBlank(topicConfInfoArr[11])) {
tmpmemCacheMsgSizeInMB = Integer.parseInt(topicConfInfoArr[11]);
}
@@ -739,9 +743,20 @@ public class BrokerSyncStatusInfo {
if (!TStringUtils.isBlank(topicConfInfoArr[13])) {
tmpmemCacheFlushIntvl = Integer.parseInt(topicConfInfoArr[13]);
}
+ if (topicConfInfoArr.length > 14) {
+ if (!TStringUtils.isNotBlank(topicConfInfoArr[14])) {
+ tmpMaxMsgSize = Integer.parseInt(topicConfInfoArr[14]);
+ Tuple2<Integer, Integer> calcResult =
+ ClusterConfigHolder.calcMaxMsgSize(tmpMaxMsgSize);
+ tmpMaxMsgSize = calcResult.getF0();
+ tmpMinMemCacheSize = calcResult.getF1();
+ }
+ }
strBuffer.append(",\"memCacheMsgSizeInMB\":").append(tmpmemCacheMsgSizeInMB);
strBuffer.append(",\"memCacheMsgCntInK\":").append(tmpmemCacheMsgCntInK);
strBuffer.append(",\"memCacheFlushIntvl\":").append(tmpmemCacheFlushIntvl);
+ strBuffer.append(",\"maxMsgSize\":").append(tmpMaxMsgSize);
+ strBuffer.append(",\"minMemCacheSize\":").append(tmpMinMemCacheSize);
strBuffer.append(",\"topicStatusId\":").append(topicStatusId);
strBuffer.append("}");
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
index 8bef5ab..9a4cf04 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
@@ -25,11 +25,13 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+
import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.cluster.BrokerInfo;
import org.apache.tubemq.corebase.cluster.TopicInfo;
+import org.apache.tubemq.corebase.utils.SettingValidUtils;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.TStatusConstants;
@@ -212,6 +214,11 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
WebParameterUtils.validIntDataParameter("memCacheFlushIntvl",
req.getParameter("memCacheFlushIntvl"),
false, defmemCacheFlushIntvl, 4000);
+ int maxMsgSizeInMB =
+ WebParameterUtils.validIntDataParameter("maxMsgSizeInMB",
+ req.getParameter("maxMsgSizeInMB"),
+ false, TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB,
+ TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB);
String attributes = strBuffer.append(TokenConstants.TOKEN_STORE_NUM)
.append(TokenConstants.EQ).append(numTopicStores)
.append(TokenConstants.SEGMENT_SEP).append(TokenConstants.TOKEN_DATA_UNFLUSHHOLD)
@@ -221,7 +228,11 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
.append(TokenConstants.SEGMENT_SEP).append(TokenConstants.TOKEN_MCACHE_MSG_SIZE)
.append(TokenConstants.EQ).append(memCacheMsgSizeInMB)
.append(TokenConstants.SEGMENT_SEP).append(TokenConstants.TOKEN_MCACHE_FLUSH_INTVL)
- .append(TokenConstants.EQ).append(memCacheFlushIntvl).toString();
+ .append(TokenConstants.EQ).append(memCacheFlushIntvl)
+ .append(TokenConstants.SEGMENT_SEP).append(TServerConstants.TOKEN_MAX_MSG_SIZE)
+ .append(TokenConstants.EQ)
+ .append(SettingValidUtils.validAndXfeMaxMsgSizeFromMBtoB(maxMsgSizeInMB))
+ .toString();
strBuffer.delete(0, strBuffer.length());
for (String itemTopicName : batchAddTopicNames) {
batchAddBdbTopicEntities.add(new BdbTopicConfEntity(oldEntity.getBrokerId(),
@@ -359,6 +370,11 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
WebParameterUtils.validIntDataParameter("memCacheFlushIntvl",
jsonObject.get("memCacheFlushIntvl"),
false, brokerConfEntity.getDftMemCacheFlushIntvl(), 4000);
+ int maxMsgSizeInMB =
+ WebParameterUtils.validIntDataParameter("maxMsgSizeInMB",
+ jsonObject.get("maxMsgSizeInMB"),
+ false, TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB,
+ TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB);
String itemCreateUser =
WebParameterUtils.validStringParameter("createUser",
jsonObject.get("createUser"),
@@ -381,7 +397,11 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
.append(TokenConstants.SEGMENT_SEP).append(TokenConstants.TOKEN_MCACHE_MSG_SIZE)
.append(TokenConstants.EQ).append(memCacheMsgSizeInMB)
.append(TokenConstants.SEGMENT_SEP).append(TokenConstants.TOKEN_MCACHE_FLUSH_INTVL)
- .append(TokenConstants.EQ).append(memCacheFlushIntvl).toString();
+ .append(TokenConstants.EQ).append(memCacheFlushIntvl)
+ .append(TokenConstants.SEGMENT_SEP).append(TServerConstants.TOKEN_MAX_MSG_SIZE)
+ .append(TokenConstants.EQ)
+ .append(SettingValidUtils.validAndXfeMaxMsgSizeFromMBtoB(maxMsgSizeInMB))
+ .toString();
strBuffer.delete(0, strBuffer.length());
batchAddItemKeys.add(inputKey);
batchAddBdbTopicEntities.add(new BdbTopicConfEntity(brokerConfEntity.getBrokerId(),
@@ -595,6 +615,8 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
.append(",\"memCacheMsgSizeInMB\":").append(entity.getMemCacheMsgSizeInMB())
.append(",\"memCacheFlushIntvl\":").append(entity.getMemCacheFlushIntvl())
.append(",\"memCacheMsgCntInK\":").append(entity.getMemCacheMsgCntInK())
+ .append(",\"maxMsgSizeInMB\":")
+ .append(entity.getMaxMsgSize() / TBaseConstants.META_MB_UNIT_SIZE)
.append(",\"createUser\":\"").append(entity.getCreateUser())
.append("\",\"createDate\":\"").append(formatter.format(entity.getCreateDate()))
.append("\",\"modifyUser\":\"").append(entity.getModifyUser())
@@ -1334,13 +1356,19 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
int memCacheMsgSizeInMB =
WebParameterUtils.validIntDataParameter("memCacheMsgSizeInMB",
req.getParameter("memCacheMsgSizeInMB"), false, TBaseConstants.META_VALUE_UNDEFINED, 2);
- memCacheMsgSizeInMB = memCacheMsgSizeInMB >= 2048 ? 2048 : memCacheMsgSizeInMB;
+ memCacheMsgSizeInMB = Math.min(memCacheMsgSizeInMB, 2048);
int memCacheFlushIntvl =
WebParameterUtils.validIntDataParameter("memCacheFlushIntvl",
req.getParameter("memCacheFlushIntvl"), false, TBaseConstants.META_VALUE_UNDEFINED, 4000);
int unFlushDataHold =
WebParameterUtils.validIntDataParameter("unflushDataHold",
req.getParameter("unflushDataHold"), false, TBaseConstants.META_VALUE_UNDEFINED, 0);
+ int maxMsgSizeInMB =
+ WebParameterUtils.validIntDataParameter("maxMsgSizeInMB",
+ req.getParameter("maxMsgSizeInMB"),
+ false, TBaseConstants.META_VALUE_UNDEFINED,
+ TBaseConstants.META_MIN_ALLOWED_MESSAGE_SIZE_MB);
+
List<BdbTopicConfEntity> batchModBdbTopicEntities = new ArrayList<>();
for (BdbBrokerConfEntity tgtEntity : batchBrokerEntitySet) {
if (tgtEntity == null) {
@@ -1418,6 +1446,15 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
newEntity.appendAttributes(TokenConstants.TOKEN_MCACHE_MSG_SIZE,
String.valueOf(memCacheMsgSizeInMB));
}
+ if (maxMsgSizeInMB > 0) {
+ int maxMsgSizeInB =
+ SettingValidUtils.validAndXfeMaxMsgSizeFromMBtoB(maxMsgSizeInMB);
+ if (maxMsgSizeInB != oldEntity.getMaxMsgSize()) {
+ foundChange = true;
+ newEntity.appendAttributes(TServerConstants.TOKEN_MAX_MSG_SIZE,
+ String.valueOf(maxMsgSizeInB));
+ }
+ }
if (memCacheFlushIntvl >= 0 && memCacheFlushIntvl != oldEntity.getMemCacheFlushIntvl()) {
foundChange = true;
newEntity.appendAttributes(TokenConstants.TOKEN_MCACHE_FLUSH_INTVL,
[incubator-tubemq] 06/49: [TUBEMQ-428] Bumped version to
0.8.0-SNAPSHOT (#341)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit ea901b79000b5f197b274ada07aebb1231feeceb
Author: dockerzhang <do...@apache.org>
AuthorDate: Fri Dec 4 17:37:34 2020 +0800
[TUBEMQ-428] Bumped version to 0.8.0-SNAPSHOT (#341)
Co-authored-by: dockerzhang <do...@tencent.com>
---
pom.xml | 2 +-
tubemq-client/pom.xml | 2 +-
tubemq-connectors/pom.xml | 2 +-
tubemq-connectors/tubemq-connector-flink/pom.xml | 2 +-
tubemq-connectors/tubemq-connector-flume/pom.xml | 2 +-
tubemq-connectors/tubemq-connector-spark/pom.xml | 2 +-
tubemq-core/pom.xml | 2 +-
tubemq-docker/pom.xml | 2 +-
tubemq-docker/tubemq-all/pom.xml | 2 +-
tubemq-docker/tubemq-build/pom.xml | 2 +-
tubemq-example/pom.xml | 2 +-
tubemq-server/pom.xml | 2 +-
12 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/pom.xml b/pom.xml
index 490fe15..bfc2b64 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
<inceptionYear>2013</inceptionYear>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache TubeMQ</name>
diff --git a/tubemq-client/pom.xml b/tubemq-client/pom.xml
index b3ffe7e..4d6f07d 100644
--- a/tubemq-client/pom.xml
+++ b/tubemq-client/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tubemq-client</artifactId>
diff --git a/tubemq-connectors/pom.xml b/tubemq-connectors/pom.xml
index 40f3e19..f8c40b8 100644
--- a/tubemq-connectors/pom.xml
+++ b/tubemq-connectors/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/tubemq-connectors/tubemq-connector-flink/pom.xml b/tubemq-connectors/tubemq-connector-flink/pom.xml
index 7a56a87..fe5568e 100644
--- a/tubemq-connectors/tubemq-connector-flink/pom.xml
+++ b/tubemq-connectors/tubemq-connector-flink/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>tubemq-connectors</artifactId>
<groupId>org.apache.tubemq</groupId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Apache TubeMQ - Connectors-flink</name>
diff --git a/tubemq-connectors/tubemq-connector-flume/pom.xml b/tubemq-connectors/tubemq-connector-flume/pom.xml
index 745a77e..c2b2f0a 100644
--- a/tubemq-connectors/tubemq-connector-flume/pom.xml
+++ b/tubemq-connectors/tubemq-connector-flume/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>tubemq-connectors</artifactId>
<groupId>org.apache.tubemq</groupId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Apache TubeMQ - Connectors-flume</name>
diff --git a/tubemq-connectors/tubemq-connector-spark/pom.xml b/tubemq-connectors/tubemq-connector-spark/pom.xml
index beb36fc..215d99b 100644
--- a/tubemq-connectors/tubemq-connector-spark/pom.xml
+++ b/tubemq-connectors/tubemq-connector-spark/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>tubemq-connectors</artifactId>
<groupId>org.apache.tubemq</groupId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Apache TubeMQ - Connectors-spark</name>
diff --git a/tubemq-core/pom.xml b/tubemq-core/pom.xml
index 9316d90..21b1bf0 100644
--- a/tubemq-core/pom.xml
+++ b/tubemq-core/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tubemq-core</artifactId>
diff --git a/tubemq-docker/pom.xml b/tubemq-docker/pom.xml
index 610ccf6..596d7e2 100644
--- a/tubemq-docker/pom.xml
+++ b/tubemq-docker/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/tubemq-docker/tubemq-all/pom.xml b/tubemq-docker/tubemq-all/pom.xml
index 8cfcf04..d265142 100644
--- a/tubemq-docker/tubemq-all/pom.xml
+++ b/tubemq-docker/tubemq-all/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>tubemq-docker</artifactId>
<groupId>org.apache.tubemq</groupId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Apache TubeMQ - Docker All</name>
diff --git a/tubemq-docker/tubemq-build/pom.xml b/tubemq-docker/tubemq-build/pom.xml
index 1eb154c..e287c73 100644
--- a/tubemq-docker/tubemq-build/pom.xml
+++ b/tubemq-docker/tubemq-build/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>tubemq-docker</artifactId>
<groupId>org.apache.tubemq</groupId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<name>Apache TubeMQ - Docker Build</name>
diff --git a/tubemq-example/pom.xml b/tubemq-example/pom.xml
index 47aa921..319acab 100644
--- a/tubemq-example/pom.xml
+++ b/tubemq-example/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>tubemq-example</artifactId>
diff --git a/tubemq-server/pom.xml b/tubemq-server/pom.xml
index 4ca762a..a4b6d39 100644
--- a/tubemq-server/pom.xml
+++ b/tubemq-server/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq</artifactId>
- <version>0.7.0-incubating-SNAPSHOT</version>
+ <version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
[incubator-tubemq] 38/49: add change
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 72213f24b55f21d340b7cb9e31b06118a333cb9d
Author: jianxzhang <ji...@tencent.com>
AuthorDate: Wed Jan 13 20:20:27 2021 +0800
add change
---
tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt b/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt
index 83d0ba5..b634af2 100644
--- a/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt
+++ b/tubemq-client-twins/tubemq-client-cpp/CMakeLists.txt
@@ -24,7 +24,7 @@ project (TubeMQ)
find_package(Protobuf REQUIRED)
-SET(CMAKE_CXX_FLAGS "-std=c++11 -O2 -g -Wall -Wsign-compare -Wfloat-equal -fno-strict-aliasing -fPIC -DASIO_STANDALONE")
+SET(CMAKE_CXX_FLAGS "-std=c++11 -O2 -g -Wall -Werror -Wno-error=class-memaccess -Wsign-compare -Wfloat-equal -fno-strict-aliasing -fPIC -DASIO_STANDALONE")
INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/proto)
[incubator-tubemq] 01/49: [TUBEMQ-433] add tubemq
perf-consumer/producer scripts (#330)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 45f70fc0d875362af51d695d5cb5c2765d34408f
Author: Yuanbo Liu <yu...@apache.org>
AuthorDate: Thu Dec 3 19:12:41 2020 +0800
[TUBEMQ-433] add tubemq perf-consumer/producer scripts (#330)
---
bin/tubemq-consumer-perf-test.sh | 40 +++++++
bin/tubemq-producer-perf-test.sh | 40 +++++++
pom.xml | 2 +-
tubemq-example/pom.xml | 10 ++
tubemq-example/src/main/assembly/assembly.xml | 3 +
.../apache/tubemq/example/ArgsParserHelper.java | 48 ++++++++
.../tubemq/example/MAMessageProducerExample.java | 100 +++++++++++------
.../tubemq/example/MessageConsumerExample.java | 125 ++++++++++++++-------
.../org/apache/tubemq/server/tools/ToolUtils.java | 4 +-
9 files changed, 295 insertions(+), 77 deletions(-)
diff --git a/bin/tubemq-consumer-perf-test.sh b/bin/tubemq-consumer-perf-test.sh
new file mode 100644
index 0000000..54189a2
--- /dev/null
+++ b/bin/tubemq-consumer-perf-test.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "$BASE_DIR" ] ; then
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+ BASE_DIR=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ BASE_DIR=`cd "$BASE_DIR" && pwd`
+ #echo "TubeMQ master is at $BASE_DIR"
+fi
+source $BASE_DIR/bin/env.sh
+$JAVA $TOOLS_ARGS org.apache.tubemq.example.MessageConsumerExample $@
diff --git a/bin/tubemq-producer-perf-test.sh b/bin/tubemq-producer-perf-test.sh
new file mode 100644
index 0000000..ed76c3a
--- /dev/null
+++ b/bin/tubemq-producer-perf-test.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "$BASE_DIR" ] ; then
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+ BASE_DIR=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ BASE_DIR=`cd "$BASE_DIR" && pwd`
+ #echo "TubeMQ master is at $BASE_DIR"
+fi
+source $BASE_DIR/bin/env.sh
+$JAVA $TOOLS_ARGS org.apache.tubemq.example.MAMessageProducerExample $@
diff --git a/pom.xml b/pom.xml
index 5a6fe4f..490fe15 100644
--- a/pom.xml
+++ b/pom.xml
@@ -312,7 +312,7 @@
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
- <version>1.2</version>
+ <version>1.4</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
diff --git a/tubemq-example/pom.xml b/tubemq-example/pom.xml
index 1095f7b..47aa921 100644
--- a/tubemq-example/pom.xml
+++ b/tubemq-example/pom.xml
@@ -65,6 +65,16 @@
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/tubemq-example/src/main/assembly/assembly.xml b/tubemq-example/src/main/assembly/assembly.xml
index 11edd32..596af24 100644
--- a/tubemq-example/src/main/assembly/assembly.xml
+++ b/tubemq-example/src/main/assembly/assembly.xml
@@ -32,6 +32,9 @@
<directory>../</directory>
<includes>
<include>./conf/tools.log4j.properties</include>
+ <include>./bin/tubemq-consumer-perf-test.sh</include>
+ <include>./bin/tubemq-producer-perf-test.sh</include>
+ <include>./bin/env.sh</include>
<include>LICENSE</include>
<include>NOTICE</include>
<include>DISCLAIMER-WIP</include>
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/ArgsParserHelper.java b/tubemq-example/src/main/java/org/apache/tubemq/example/ArgsParserHelper.java
new file mode 100644
index 0000000..c507ae4
--- /dev/null
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/ArgsParserHelper.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.example;
+
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+
+public class ArgsParserHelper {
+
+ /**
+ * Print help information and exit.
+ *
+ * @param opts - options
+ */
+ public static void help(String commandName, Options opts) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(commandName, opts);
+ System.exit(0);
+ }
+
+ /**
+ * Init common options when parsing args.
+ * @return - options
+ */
+ public static Options initCommonOptions() {
+ Options options = new Options();
+ options.addOption(null, "help", false, "show help");
+ options.addOption(null, "master-list", true, "master address like: host1:8000,host2:8000");
+ options.addOption(null, "topic", true, "topic list, topic1,topic2 or "
+ + "topic1:tid11;tid12,topic2:tid21;tid22(consumer only)");
+ return options;
+ }
+}
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
index f76b077..9c76ce1 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
@@ -30,8 +30,13 @@ import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Options;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.tubemq.client.config.TubeClientConfig;
import org.apache.tubemq.client.exception.TubeClientException;
@@ -59,7 +64,7 @@ public class MAMessageProducerExample {
private static final int SESSION_FACTORY_NUM = 10;
private static Set<String> topicSet;
- private static int msgCount;
+ private static int batchCount;
private static int producerCount;
private static byte[] sendData;
@@ -89,45 +94,72 @@ public class MAMessageProducerExample {
}
}
- public static void main(String[] args) {
- final String masterHostAndPort = args[0];
-
- final String topics = args[1];
- final List<String> topicList = Arrays.asList(topics.split(","));
-
- topicSet = new TreeSet<>(topicList);
-
- msgCount = Integer.parseInt(args[2]);
- producerCount = Math.min(args.length > 4 ? Integer.parseInt(args[3]) : 10, MAX_PRODUCER_NUM);
-
- logger.info("MAMessageProducerExample.main started...");
+ /**
+ * Init options
+ *
+ * @return options
+ */
+ public static Options initOptions() {
+ Options options = ArgsParserHelper.initCommonOptions();
+ options.addOption(null, "batch-size", true, "number of messages in single batch, default is 100000");
+ options.addOption(null, "max-batch", true, "max batch number, default is 1024");
+ options.addOption(null, "thread-num", true, "thread number of producers, default is 1, max is 100");
+ return options;
+ }
- final byte[] transmitData = StringUtils.getBytesUtf8("This is a test message from multi-session factory.");
- final ByteBuffer dataBuffer = ByteBuffer.allocate(1024);
+ public static void main(String[] args) {
+ Options options = null;
+ try {
+ CommandLineParser parser = new DefaultParser();
+ options = initOptions();
+ CommandLine cl = parser.parse(options, args);
+ if (cl != null) {
+ final String masterHostAndPort = cl.getOptionValue("master-list");
+ final String topics = cl.getOptionValue("topic");
+ final List<String> topicList = Arrays.asList(topics.split(","));
+ topicSet = new TreeSet<>(topicList);
+
+ batchCount = Integer.parseInt(cl.getOptionValue("max-batch", "100000"));
+ int batchSize = Integer.parseInt(cl.getOptionValue("batch-size", "1024"));
+ producerCount = Math.min(Integer.parseInt(cl.getOptionValue(
+ "thread-num", "1")), MAX_PRODUCER_NUM);
+ logger.info("MAMessageProducerExample.main started...");
+ final byte[] transmitData = StringUtils
+ .getBytesUtf8("This is a test message from multi-session factory.");
+ final ByteBuffer dataBuffer = ByteBuffer.allocate(batchSize);
+
+ while (dataBuffer.hasRemaining()) {
+ int offset = dataBuffer.arrayOffset();
+ dataBuffer.put(transmitData, offset,
+ Math.min(dataBuffer.remaining(), transmitData.length));
+ }
- while (dataBuffer.hasRemaining()) {
- int offset = dataBuffer.arrayOffset();
- dataBuffer.put(transmitData, offset, Math.min(dataBuffer.remaining(), transmitData.length));
- }
+ dataBuffer.flip();
+ sendData = dataBuffer.array();
- dataBuffer.flip();
- sendData = dataBuffer.array();
+ try {
+ MAMessageProducerExample messageProducer = new MAMessageProducerExample(
+ masterHostAndPort);
- try {
- MAMessageProducerExample messageProducer = new MAMessageProducerExample(masterHostAndPort);
+ messageProducer.startService();
- messageProducer.startService();
+ while (SENT_SUCC_COUNTER.get() < (long) batchCount * producerCount * topicSet.size()) {
+ TimeUnit.MILLISECONDS.sleep(1000);
+ }
+ messageProducer.producerMap.clear();
+ messageProducer.shutdown();
- while (SENT_SUCC_COUNTER.get() < msgCount * producerCount * topicSet.size()) {
- Thread.sleep(1000);
+ } catch (TubeClientException e) {
+ logger.error("TubeClientException: ", e);
+ } catch (Throwable e) {
+ logger.error("Throwable: ", e);
+ }
+ }
+ } catch (Exception ex) {
+ logger.error(ex.getMessage());
+ if (options != null) {
+ ArgsParserHelper.help("./tubemq-producer-perf-test.sh", options);
}
- messageProducer.producerMap.clear();
- messageProducer.shutdown();
-
- } catch (TubeClientException e) {
- logger.error("TubeClientException: ", e);
- } catch (Throwable e) {
- logger.error("Throwable: ", e);
}
}
@@ -173,7 +205,7 @@ public class MAMessageProducerExample {
} catch (Throwable t) {
logger.error("publish exception: ", t);
}
- for (int i = 0; i < msgCount; i++) {
+ for (int i = 0; i < batchCount; i++) {
long millis = System.currentTimeMillis();
for (String topic : topicSet) {
try {
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
index d9aeb8a..3b99943 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
@@ -26,6 +26,10 @@ import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Options;
import org.apache.tubemq.client.common.PeerInfo;
import org.apache.tubemq.client.config.ConsumerConfig;
import org.apache.tubemq.client.consumer.ConsumePosition;
@@ -59,29 +63,45 @@ public final class MessageConsumerExample {
private static final MsgRecvStats msgRecvStats = new MsgRecvStats();
private final PushMessageConsumer messageConsumer;
- private final MessageSessionFactory messageSessionFactory;
- public MessageConsumerExample(String masterHostAndPort, String group, int fetchCount) throws Exception {
+ public MessageConsumerExample(String masterHostAndPort, String group,
+ int fetchCount, boolean isFromBegin) throws Exception {
ConsumerConfig consumerConfig = new ConsumerConfig(masterHostAndPort, group);
- consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_LATEST_OFFSET);
+ if (isFromBegin) {
+ consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_FIRST_OFFSET);
+ } else {
+ consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_LATEST_OFFSET);
+ }
if (fetchCount > 0) {
consumerConfig.setPushFetchThreadCnt(fetchCount);
}
- this.messageSessionFactory = new TubeSingleSessionFactory(consumerConfig);
+ MessageSessionFactory messageSessionFactory = new TubeSingleSessionFactory(consumerConfig);
this.messageConsumer = messageSessionFactory.createPushConsumer(consumerConfig);
}
- public static void main(String[] args) {
- final String masterHostAndPort = args[0];
- final String topics = args[1];
- final String group = args[2];
- final int consumerCount = Integer.parseInt(args[3]);
- int fetchCount = -1;
- if (args.length > 5) {
- fetchCount = Integer.parseInt(args[4]);
- }
- final Map<String, TreeSet<String>> topicTidsMap = new HashMap<>();
+ /**
+ * Init options
+ * @return options
+ */
+ public static Options initOptions() {
+
+ Options options = ArgsParserHelper.initCommonOptions();
+ options.addOption(null, "batch-size", true, "max number of fetching message in one batch");
+ options.addOption(null, "thread-num", true, "thread number of consumers");
+ options.addOption(null, "group", true, "consumer group");
+ options.addOption(null, "from-begin", false, "default is consuming from latest, "
+ + "if option is clarified, then consume from begin");
+ return options;
+ }
+
+ /**
+ * init topic->set(tid) map
+ * @param topics - topics string
+ * @return - map of topic->set(tid)
+ */
+ private static Map<String, TreeSet<String>> initTopicList(String topics) {
+ Map<String, TreeSet<String>> topicTidsMap = new HashMap<>();
String[] topicTidsList = topics.split(",");
for (String topicTids : topicTidsList) {
String[] topicTidStr = topicTids.split(":");
@@ -95,35 +115,60 @@ public final class MessageConsumerExample {
}
topicTidsMap.put(topicTidStr[0], tids);
}
- final int startFetchCount = fetchCount;
- final ExecutorService executorService = Executors.newFixedThreadPool(fetchCount);
- for (int i = 0; i < consumerCount; i++) {
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- try {
- MessageConsumerExample messageConsumer = new MessageConsumerExample(
- masterHostAndPort,
- group,
- startFetchCount
- );
- messageConsumer.subscribe(topicTidsMap);
- } catch (Exception e) {
- logger.error("Create consumer failed!", e);
- }
- }
- });
- }
- final Thread statisticThread = new Thread(msgRecvStats, "Received Statistic Thread");
- statisticThread.start();
+ return topicTidsMap;
+ }
- executorService.shutdown();
+ public static void main(String[] args) {
+ Options options = null;
try {
- executorService.awaitTermination(60 * 1000, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- logger.error("Thread Pool shutdown has been interrupted!");
+ CommandLineParser parser = new DefaultParser();
+ options = initOptions();
+ CommandLine cl = parser.parse(options, args);
+ if (cl != null) {
+ final String masterHostAndPort = cl.getOptionValue("master-list");
+ final Map<String, TreeSet<String>> topicTidsMap = initTopicList(
+ cl.getOptionValue("topic"));
+ final String group = cl.getOptionValue("group");
+ int threadNum = Integer.parseInt(cl.getOptionValue("thread-num", "1"));
+ final int fetchCount = Integer.parseInt(cl.getOptionValue("batch-size", "-1"));
+ final boolean isFromBegin = cl.hasOption("from-begin");
+ ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
+ for (int i = 0; i < threadNum; i++) {
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ MessageConsumerExample messageConsumer = new MessageConsumerExample(
+ masterHostAndPort,
+ group,
+ fetchCount,
+ isFromBegin
+ );
+ messageConsumer.subscribe(topicTidsMap);
+ } catch (Exception e) {
+ logger.error("Create consumer failed!", e);
+ }
+ }
+ });
+ }
+ final Thread statisticThread = new Thread(msgRecvStats,
+ "Received Statistic Thread");
+ statisticThread.start();
+
+ executorService.shutdown();
+ try {
+ executorService.awaitTermination(60 * 1000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ logger.error("Thread Pool shutdown has been interrupted!");
+ }
+ msgRecvStats.stopStats();
+ }
+ } catch (Exception ex) {
+ logger.error(ex.getMessage(), ex.getMessage());
+ if (options != null) {
+ ArgsParserHelper.help("./tubemq-consumer-perf-test.sh", options);
+ }
}
- msgRecvStats.stopStats();
}
public void subscribe(Map<String, TreeSet<String>> topicTidsMap) throws TubeClientException {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
index d2c503a..9c343a8 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/ToolUtils.java
@@ -19,10 +19,10 @@ package org.apache.tubemq.server.tools;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.PosixParser;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.broker.BrokerConfig;
import org.apache.tubemq.server.broker.exception.StartupException;
@@ -56,7 +56,7 @@ public class ToolUtils {
final Options options = new Options();
final Option file = new Option("f", true, "configuration file path");
options.addOption(file);
- final CommandLineParser parser = new PosixParser();
+ final CommandLineParser parser = new DefaultParser();
CommandLine line = null;
try {
line = parser.parse(options, args);
[incubator-tubemq] 10/49: [TUBEMQ-445]Adjust the status check
default sleep interval of pullConsumeReadyChkSliceMs (#345)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 7a1d7cba35fe8b60cb2767ee11e7d5d1138fdaae
Author: gosonzhang <46...@qq.com>
AuthorDate: Wed Dec 9 18:25:03 2020 +0800
[TUBEMQ-445]Adjust the status check default sleep interval of pullConsumeReadyChkSliceMs (#345)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../main/java/org/apache/tubemq/client/common/TClientConstants.java | 2 +-
.../org/apache/tubemq/client/consumer/SimplePullMessageConsumer.java | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/common/TClientConstants.java b/tubemq-client/src/main/java/org/apache/tubemq/client/common/TClientConstants.java
index 36808c0..044fa17 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/common/TClientConstants.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/common/TClientConstants.java
@@ -25,7 +25,7 @@ public class TClientConstants {
public static final long CFG_DEFAULT_REGFAIL_WAIT_PERIOD_MS = 1000;
public static final long CFG_DEFAULT_MSG_NOTFOUND_WAIT_PERIOD_MS = 400L;
public static final long CFG_DEFAULT_CONSUME_READ_WAIT_PERIOD_MS = 90000L;
- public static final long CFG_DEFAULT_CONSUME_READ_CHECK_SLICE_MS = 300L;
+ public static final long CFG_DEFAULT_CONSUME_READ_CHECK_SLICE_MS = 50L;
public static final long CFG_DEFAULT_PUSH_LISTENER_WAIT_PERIOD_MS = 3000L;
public static final long CFG_DEFAULT_PULL_REB_CONFIRM_WAIT_PERIOD_MS = 3000L;
public static final long CFG_DEFAULT_PULL_PROTECT_CONFIRM_WAIT_PERIOD_MS = 60000L;
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/SimplePullMessageConsumer.java b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/SimplePullMessageConsumer.java
index b0fc8b9..6c1146d 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/SimplePullMessageConsumer.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/SimplePullMessageConsumer.java
@@ -147,7 +147,9 @@ public class SimplePullMessageConsumer implements PullMessageConsumer {
baseConsumer.getConsumerConfig().getPullConsumeReadyWaitPeriodMs())) {
return new ConsumerResult(selectResult.getErrCode(), selectResult.getErrMsg());
}
- ThreadUtils.sleep(baseConsumer.getConsumerConfig().getPullConsumeReadyChkSliceMs());
+ if (baseConsumer.getConsumerConfig().getPullConsumeReadyChkSliceMs() > 10) {
+ ThreadUtils.sleep(baseConsumer.getConsumerConfig().getPullConsumeReadyChkSliceMs());
+ }
}
StringBuilder sBuilder = new StringBuilder(512);
// Check the data cache first
[incubator-tubemq] 28/49: [TUBEMQ-499] Add configure store
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit f5ae304281370ac17968fc4fe6e20d0c20f3b47d
Author: gosonzhang <go...@tencent.com>
AuthorDate: Fri Jan 8 18:41:23 2021 +0800
[TUBEMQ-499] Add configure store
---
.../apache/tubemq/corebase/utils/MixedUtils.java | 9 +
tubemq-core/src/main/proto/MasterService.proto | 18 ++
.../tubemq/server/common/TServerConstants.java | 2 +
.../server/master/bdbstore/BdbStoreService.java | 7 +
.../master/bdbstore/DefaultBdbStoreService.java | 95 +++++++
.../bdbentitys/BdbClusterSettingEntity.java | 309 +++++++++++++++++++++
.../nodemanage/nodebroker/BrokerConfManager.java | 95 +++++++
7 files changed, 535 insertions(+)
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
index bfbedad..bcd0738 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
@@ -92,4 +92,13 @@ public class MixedUtils {
dataBuffer.flip();
return dataBuffer.array();
}
+
+ // get the middle data between min, max, and data
+ public static int mid(int data, int min, int max) {
+ return Math.max(min, Math.min(max, data));
+ }
+
+ public static long mid(long data, long min, long max) {
+ return Math.max(min, Math.min(max, data));
+ }
}
diff --git a/tubemq-core/src/main/proto/MasterService.proto b/tubemq-core/src/main/proto/MasterService.proto
index cefd2d2..27e5496 100644
--- a/tubemq-core/src/main/proto/MasterService.proto
+++ b/tubemq-core/src/main/proto/MasterService.proto
@@ -61,6 +61,16 @@ message MasterBrokerAuthorizedInfo {
optional string authAuthorizedToken = 2;
}
+message ApprovedClientConfig {
+ required int64 configId = 1;
+ optional int32 maxMsgSize = 2;
+}
+
+message ClusterDefConfig {
+ required int64 configId = 1;
+ optional int32 maxMsgSize = 2;
+}
+
message RegisterRequestP2M {
required string clientId = 1;
repeated string topicList = 2;
@@ -68,6 +78,7 @@ message RegisterRequestP2M {
required string hostName = 4;
optional MasterCertificateInfo authInfo = 5;
optional string jdkVersion = 6;
+ optional ApprovedClientConfig appdConfig = 7;
}
message RegisterResponseM2P {
@@ -77,6 +88,7 @@ message RegisterResponseM2P {
required int64 brokerCheckSum = 4;
repeated string brokerInfos = 5;
optional MasterAuthorizedInfo authorizedInfo = 6;
+ optional ApprovedClientConfig appdConfig = 7;
}
message HeartRequestP2M {
@@ -85,6 +97,7 @@ message HeartRequestP2M {
required string hostName = 3;
repeated string topicList = 4;
optional MasterCertificateInfo authInfo = 5;
+ optional ApprovedClientConfig appdConfig = 6;
}
message HeartResponseM2P {
@@ -97,6 +110,7 @@ message HeartResponseM2P {
repeated string brokerInfos = 6;
optional bool requireAuth = 7;
optional MasterAuthorizedInfo authorizedInfo = 8;
+ optional ApprovedClientConfig appdConfig = 9;
}
message CloseRequestP2M{
@@ -208,6 +222,7 @@ message RegisterRequestB2M {
optional int32 qryPriorityId = 12;
optional int32 tlsPort = 13;
optional MasterCertificateInfo authInfo = 14;
+ optional ClusterDefConfig clsDefConfig = 15;
}
message RegisterResponseM2B {
@@ -230,6 +245,7 @@ message RegisterResponseM2B {
optional int32 qryPriorityId = 15;
optional MasterAuthorizedInfo authorizedInfo = 16; /* Deprecated */
optional MasterBrokerAuthorizedInfo brokerAuthorizedInfo = 17;
+ optional ClusterDefConfig clsDefConfig = 18;
}
message HeartRequestB2M {
@@ -250,6 +266,7 @@ message HeartRequestB2M {
optional int64 flowCheckId = 13;
optional int32 qryPriorityId = 14;
optional MasterCertificateInfo authInfo = 15;
+ optional ClusterDefConfig clsDefConfig = 16;
}
message HeartResponseM2B {
@@ -275,6 +292,7 @@ message HeartResponseM2B {
optional int32 qryPriorityId = 17;
optional MasterAuthorizedInfo authorizedInfo = 18; /* Deprecated */
optional MasterBrokerAuthorizedInfo brokerAuthorizedInfo = 19;
+ optional ClusterDefConfig clsDefConfig = 20;
}
message CloseRequestB2M {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java
index 12af51a..5793364 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/TServerConstants.java
@@ -25,6 +25,8 @@ public final class TServerConstants {
public static final String TOKEN_JOB_TOPICS = "topics";
public static final String TOKEN_JOB_STORE_MGR = "messageStoreManager";
public static final String TOKEN_DEFAULT_FLOW_CONTROL = "default_master_ctrl";
+ public static final String TOKEN_DEFAULT_CLUSTER_SETTING = "default_cluster_config";
+
public static final String TOKEN_BLANK_FILTER_CONDITION = ",,";
public static final int CFG_MODAUTHTOKEN_MAX_LENGTH = 128;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/BdbStoreService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/BdbStoreService.java
index 7b264c7..4cb2185 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/BdbStoreService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/BdbStoreService.java
@@ -21,6 +21,7 @@ import java.net.InetSocketAddress;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBlackGroupEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbClusterSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumeGroupSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumerGroupEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbGroupFilterCondEntity;
@@ -103,4 +104,10 @@ public interface BdbStoreService {
boolean isNew);
ConcurrentHashMap<String, BdbConsumeGroupSettingEntity> getConsumeGroupSettingMap();
+
+ boolean putBdbClusterConfEntity(BdbClusterSettingEntity clusterConfEntity, boolean isNew);
+
+ boolean delBdbClusterConfEntity();
+
+ ConcurrentHashMap<String, BdbClusterSettingEntity> getClusterDefSettingMap();
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java
index b201cde..1b8a1b1 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/DefaultBdbStoreService.java
@@ -55,11 +55,13 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.Server;
+import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.fileconfig.MasterReplicationConfig;
import org.apache.tubemq.server.master.MasterConfig;
import org.apache.tubemq.server.master.TMaster;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBlackGroupEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbClusterSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumeGroupSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumerGroupEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbGroupFilterCondEntity;
@@ -72,6 +74,7 @@ import org.apache.tubemq.server.master.web.model.ClusterNodeVO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* Bdb store service
* like a local database manager, according to database table name, store instance, primary key, memory cache
@@ -80,6 +83,7 @@ import org.slf4j.LoggerFactory;
public class DefaultBdbStoreService implements BdbStoreService, Server {
private static final Logger logger = LoggerFactory.getLogger(DefaultBdbStoreService.class);
+ private static final String BDB_CLUSTER_SETTING_STORE_NAME = "bdbClusterSetting";
private static final String BDB_TOPIC_CONFIG_STORE_NAME = "bdbTopicConfig";
private static final String BDB_BROKER_CONFIG_STORE_NAME = "bdbBrokerConfig";
private static final String BDB_CONSUMER_GROUP_STORE_NAME = "bdbConsumerGroup";
@@ -152,6 +156,11 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
private PrimaryIndex<String/* recordKey */, BdbConsumeGroupSettingEntity> consumeGroupSettingIndex;
private ConcurrentHashMap<String/* consumeGroup */, BdbConsumeGroupSettingEntity> consumeGroupSettingMap =
new ConcurrentHashMap<>();
+ // cluster default setting store
+ private EntityStore clusterDefSettingStore;
+ private PrimaryIndex<String/* recordKey */, BdbClusterSettingEntity> clusterDefSettingIndex;
+ private ConcurrentHashMap<String/* recordKey */, BdbClusterSettingEntity> clusterDefSettingMap =
+ new ConcurrentHashMap<>();
// service status
private AtomicBoolean isStarted = new AtomicBoolean(false);
// master role flag
@@ -386,6 +395,14 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
logger.error("[BDB Error] Close groupFlowCtrlStore error ", e);
}
}
+ if (clusterDefSettingStore != null) {
+ try {
+ clusterDefSettingStore.close();
+ clusterDefSettingStore = null;
+ } catch (Throwable e) {
+ logger.error("[BDB Error] Close clusterDefSettingStore error ", e);
+ }
+ }
/* evn close */
if (repEnv != null) {
try {
@@ -769,6 +786,39 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
return true;
}
+ /**
+ * Put cluster default setting bdb entity
+ *
+ * @param clusterConfEntity
+ * @param isNew
+ * @return
+ */
+ @Override
+ public boolean putBdbClusterConfEntity(BdbClusterSettingEntity clusterConfEntity, boolean isNew) {
+ BdbClusterSettingEntity result = null;
+ try {
+ result = clusterDefSettingIndex.put(clusterConfEntity);
+ } catch (Throwable e) {
+ logger.error("[BDB Error] Put ClusterConfEntity Error ", e);
+ return false;
+ }
+ if (isNew) {
+ return result == null;
+ }
+ return result != null;
+ }
+
+ @Override
+ public boolean delBdbClusterConfEntity() {
+ try {
+ clusterDefSettingIndex.delete(TServerConstants.TOKEN_DEFAULT_CLUSTER_SETTING);
+ } catch (Throwable e) {
+ logger.error("[BDB Error] delBdbClusterConfEntity Error ", e);
+ return false;
+ }
+ return true;
+ }
+
@Override
public ConcurrentHashMap<String,
ConcurrentHashMap<String, BdbConsumerGroupEntity>> getConsumerGroupNameAccControlMap() {
@@ -797,6 +847,11 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
return this.consumeGroupSettingMap;
}
+ @Override
+ public ConcurrentHashMap<String, BdbClusterSettingEntity> getClusterDefSettingMap() {
+ return this.clusterDefSettingMap;
+ }
+
/**
* Get master group status
*
@@ -977,6 +1032,10 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
new EntityStore(repEnv, BDB_CONSUME_GROUP_SETTING_STORE_NAME, storeConfig);
consumeGroupSettingIndex =
consumeGroupSettingStore.getPrimaryIndex(String.class, BdbConsumeGroupSettingEntity.class);
+ clusterDefSettingStore =
+ new EntityStore(repEnv, BDB_CLUSTER_SETTING_STORE_NAME, storeConfig);
+ clusterDefSettingIndex =
+ clusterDefSettingStore.getPrimaryIndex(String.class, BdbClusterSettingEntity.class);
}
/**
@@ -1394,6 +1453,41 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
logger.info("loadConsumeGroupSettingUnits successfully...");
}
+
+ private void loadClusterDefSettingUnits() throws Exception {
+ long count = 0L;
+ EntityCursor<BdbClusterSettingEntity> cursor = null;
+ logger.info("loadClusterDefSettingUnits start...");
+ try {
+ cursor = clusterDefSettingIndex.entities();
+ clusterDefSettingMap.clear();
+ StringBuilder sBuilder = logger.isDebugEnabled() ? new StringBuilder(512) : null;
+ logger.debug("[loadClusterDefSettingUnits] Load consumer group begin:");
+ for (BdbClusterSettingEntity bdbEntity : cursor) {
+ if (bdbEntity == null) {
+ logger.warn("[BDB Error] Found Null data while loading from clusterDefSettingIndex!");
+ continue;
+ }
+ clusterDefSettingMap.put(bdbEntity.getRecordKey(), bdbEntity);
+ count++;
+ if (logger.isDebugEnabled()) {
+ logger.debug(bdbEntity.toJsonString(sBuilder).toString());
+ sBuilder.delete(0, sBuilder.length());
+ }
+ }
+ logger.debug("[loadClusterDefSettingUnits] Load consumer group finished!");
+ logger.info("[loadClusterDefSettingUnits] total load records are {}", count);
+ } catch (Exception e) {
+ logger.error("[loadClusterDefSettingUnits error] ", e);
+ throw e;
+ } finally {
+ if (cursor != null) {
+ cursor.close();
+ }
+ }
+ logger.info("loadClusterDefSettingUnits successfully...");
+ }
+
public class Listener implements StateChangeListener {
@Override
public void stateChange(StateChangeEvent stateChangeEvent) throws RuntimeException {
@@ -1424,6 +1518,7 @@ public class DefaultBdbStoreService implements BdbStoreService, Server {
if (!isMaster) {
try {
clearCachedRunData();
+ loadClusterDefSettingUnits();
loadBrokerConfUnits();
loadTopicConfUnits();
loadGroupFlowCtrlUnits();
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
new file mode 100644
index 0000000..ca6e1b4
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/bdbstore/bdbentitys/BdbClusterSettingEntity.java
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.master.bdbstore.bdbentitys;
+
+import com.sleepycat.persist.model.Entity;
+import com.sleepycat.persist.model.PrimaryKey;
+import java.io.Serializable;
+import java.util.Date;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.server.common.utils.WebParameterUtils;
+
+
+/*
+ * store the cluster default setting
+ *
+ */
+@Entity
+public class BdbClusterSettingEntity implements Serializable {
+
+ private static final long serialVersionUID = -3259439355290322115L;
+
+ @PrimaryKey
+ private String recordKey = "";
+ //broker tcp port
+ private int brokerPort = TBaseConstants.META_VALUE_UNDEFINED;
+ //broker tls port
+ private int brokerTLSPort = TBaseConstants.META_VALUE_UNDEFINED;
+ //broker web port
+ private int brokerWebPort = TBaseConstants.META_VALUE_UNDEFINED;
+ //store num
+ private int numTopicStores = TBaseConstants.META_VALUE_UNDEFINED;
+ //partition num
+ private int numPartitions = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush disk threshold
+ private int unflushDskThreshold = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush disk interval
+ private int unflushDksInterval = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush memory cache threshold
+ private int unflushMemThreshold = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush memory cache interval
+ private int unflushMemInterval = TBaseConstants.META_VALUE_UNDEFINED;
+ //flush memory cache count
+ private int unflushMemCnt = TBaseConstants.META_VALUE_UNDEFINED;
+ private boolean acceptPublish = true; //enable publish
+ private boolean acceptSubscribe = true; //enable subscribe
+ private String deleteWhen = ""; //delete policy execute time
+ private int qryPriorityId = TBaseConstants.META_VALUE_UNDEFINED;
+ private int maxMsgSize = TBaseConstants.META_VALUE_UNDEFINED;
+ private String attributes; //extra attribute
+ private String modifyUser; //modify user
+ private Date modifyDate; //modify date
+
+ public BdbClusterSettingEntity() {
+ }
+
+ //Constructor
+ public BdbClusterSettingEntity(String recordKey, int brokerPort, int brokerTLSPort,
+ int brokerWebPort, int numTopicStores, int numPartitions,
+ int unflushDskThreshold, int unflushDksInterval,
+ int unflushMemThreshold, int unflushMemInterval,
+ int unflushMemCnt, boolean acceptPublish,
+ boolean acceptSubscribe, String deleteWhen,
+ int qryPriorityId, int maxMsgSize, String attributes,
+ String modifyUser, Date modifyDate) {
+ this.recordKey = recordKey;
+ this.brokerPort = brokerPort;
+ this.brokerTLSPort = brokerTLSPort;
+ this.brokerWebPort = brokerWebPort;
+ this.numTopicStores = numTopicStores;
+ this.numPartitions = numPartitions;
+ this.unflushDskThreshold = unflushDskThreshold;
+ this.unflushDksInterval = unflushDksInterval;
+ this.unflushMemThreshold = unflushMemThreshold;
+ this.unflushMemInterval = unflushMemInterval;
+ this.unflushMemCnt = unflushMemCnt;
+ this.acceptPublish = acceptPublish;
+ this.acceptSubscribe = acceptSubscribe;
+ this.deleteWhen = deleteWhen;
+ this.qryPriorityId = qryPriorityId;
+ this.maxMsgSize = maxMsgSize;
+ this.attributes = attributes;
+ this.modifyUser = modifyUser;
+ this.modifyDate = modifyDate;
+ }
+
+ public void setRecordKey(String recordKey) {
+ this.recordKey = recordKey;
+ }
+
+ public String getRecordKey() {
+ return recordKey;
+ }
+
+ public int getBrokerPort() {
+ return brokerPort;
+ }
+
+ public void setBrokerPort(int brokerPort) {
+ this.brokerPort = brokerPort;
+ }
+
+ public int getBrokerTLSPort() {
+ return brokerTLSPort;
+ }
+
+ public void setBrokerTLSPort(int brokerTLSPort) {
+ this.brokerTLSPort = brokerTLSPort;
+ }
+
+ public int getBrokerWebPort() {
+ return brokerWebPort;
+ }
+
+ public void setBrokerWebPort(int brokerWebPort) {
+ this.brokerWebPort = brokerWebPort;
+ }
+
+ public int getNumTopicStores() {
+ return numTopicStores;
+ }
+
+ public void setNumTopicStores(int numTopicStores) {
+ this.numTopicStores = numTopicStores;
+ }
+
+ public int getNumPartitions() {
+ return numPartitions;
+ }
+
+ public void setNumPartitions(int numPartitions) {
+ this.numPartitions = numPartitions;
+ }
+
+ public int getUnflushDskThreshold() {
+ return unflushDskThreshold;
+ }
+
+ public void setUnflushDskThreshold(int unflushDskThreshold) {
+ this.unflushDskThreshold = unflushDskThreshold;
+ }
+
+ public int getUnflushDksInterval() {
+ return unflushDksInterval;
+ }
+
+ public void setUnflushDksInterval(int unflushDksInterval) {
+ this.unflushDksInterval = unflushDksInterval;
+ }
+
+ public int getUnflushMemThreshold() {
+ return unflushMemThreshold;
+ }
+
+ public void setUnflushMemThreshold(int unflushMemThreshold) {
+ this.unflushMemThreshold = unflushMemThreshold;
+ }
+
+ public int getUnflushMemInterval() {
+ return unflushMemInterval;
+ }
+
+ public void setUnflushMemInterval(int unflushMemInterval) {
+ this.unflushMemInterval = unflushMemInterval;
+ }
+
+ public int getUnflushMemCnt() {
+ return unflushMemCnt;
+ }
+
+ public void setUnflushMemCnt(int unflushMemCnt) {
+ this.unflushMemCnt = unflushMemCnt;
+ }
+
+ public boolean isAcceptPublish() {
+ return acceptPublish;
+ }
+
+ public void setAcceptPublish(boolean acceptPublish) {
+ this.acceptPublish = acceptPublish;
+ }
+
+ public boolean isAcceptSubscribe() {
+ return acceptSubscribe;
+ }
+
+ public void setAcceptSubscribe(boolean acceptSubscribe) {
+ this.acceptSubscribe = acceptSubscribe;
+ }
+
+ public String getDeleteWhen() {
+ return deleteWhen;
+ }
+
+ public void setDeleteWhen(String deleteWhen) {
+ this.deleteWhen = deleteWhen;
+ }
+
+ public int getQryPriorityId() {
+ return qryPriorityId;
+ }
+
+ public void setQryPriorityId(int qryPriorityId) {
+ this.qryPriorityId = qryPriorityId;
+ }
+
+ public int getMaxMsgSize() {
+ return maxMsgSize;
+ }
+
+ public void setMaxMsgSize(int maxMsgSize) {
+ this.maxMsgSize = maxMsgSize;
+ }
+
+ public String getAttributes() {
+ return attributes;
+ }
+
+ public void setAttributes(String attributes) {
+ this.attributes = attributes;
+ }
+
+ public String getModifyUser() {
+ return modifyUser;
+ }
+
+ public void setModifyUser(String modifyUser) {
+ this.modifyUser = modifyUser;
+ }
+
+ public Date getModifyDate() {
+ return modifyDate;
+ }
+
+ public void setModifyDate(Date modifyDate) {
+ this.modifyDate = modifyDate;
+ }
+
+ /**
+ * Serialize field to json format
+ *
+ * @param sBuilder
+ * @return
+ */
+ public StringBuilder toJsonString(final StringBuilder sBuilder) {
+ return sBuilder.append("{\"type\":\"BdbClusterSettingEntity\",")
+ .append("\"recordKey\":\"").append(recordKey).append("\"")
+ .append(",\"brokerPort\":").append(brokerPort)
+ .append(",\"brokerTLSPort\":").append(brokerTLSPort)
+ .append(",\"brokerWebPort\":").append(brokerWebPort)
+ .append(",\"numTopicStores\":").append(numTopicStores)
+ .append(",\"numPartitions\":").append(numPartitions)
+ .append(",\"unflushDskThreshold\":").append(unflushDskThreshold)
+ .append(",\"unflushDksInterval\":").append(unflushDksInterval)
+ .append(",\"unflushMemThreshold\":").append(unflushMemThreshold)
+ .append(",\"unflushMemInterval\":").append(unflushMemInterval)
+ .append(",\"unflushMemCnt\":").append(unflushMemCnt)
+ .append(",\"acceptPublish\":").append(acceptPublish)
+ .append(",\"acceptSubscribe\":").append(acceptSubscribe)
+ .append(",\"deleteWhen\":\"").append(deleteWhen).append("\"")
+ .append(",\"maxMsgSize\":").append(maxMsgSize)
+ .append(",\"qryPriorityId\":").append(qryPriorityId)
+ .append(",\"attributes\":\"").append(attributes).append("\"")
+ .append(",\"modifyUser\":\"").append(modifyUser).append("\"")
+ .append(",\"modifyDate\":\"")
+ .append(WebParameterUtils.date2yyyyMMddHHmmss(modifyDate))
+ .append("\"}");
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this)
+ .append("recordKey", recordKey)
+ .append("brokerPort", brokerPort)
+ .append("brokerTLSPort", brokerTLSPort)
+ .append("brokerWebPort", brokerWebPort)
+ .append("numTopicStores", numTopicStores)
+ .append("numPartitions", numPartitions)
+ .append("unflushDskThreshold", unflushDskThreshold)
+ .append("unflushDksInterval", unflushDksInterval)
+ .append("unflushMemThreshold", unflushMemThreshold)
+ .append("unflushMemInterval", unflushMemInterval)
+ .append("unflushMemCnt", unflushMemCnt)
+ .append("acceptPublish", acceptPublish)
+ .append("acceptSubscribe", acceptSubscribe)
+ .append("deleteWhen", deleteWhen)
+ .append("maxMsgSize", maxMsgSize)
+ .append("qryPriorityId", qryPriorityId)
+ .append("attributes", attributes)
+ .append("modifyUser", modifyUser)
+ .append("modifyDate", modifyDate)
+ .toString();
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
index 698f0d3..86a6bc4 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
@@ -46,6 +46,7 @@ import org.apache.tubemq.server.master.bdbstore.DefaultBdbStoreService;
import org.apache.tubemq.server.master.bdbstore.MasterGroupStatus;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBlackGroupEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
+import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbClusterSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumeGroupSettingEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbConsumerGroupEntity;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbGroupFilterCondEntity;
@@ -90,6 +91,7 @@ public class BrokerConfManager implements Server {
ConcurrentHashMap<String /* consumerGroup */, BdbGroupFilterCondEntity>> groupFilterCondTopicMap;
private ConcurrentHashMap<String /* groupName */, BdbGroupFlowCtrlEntity> consumeGroupFlowCtrlMap;
private ConcurrentHashMap<String /* consumeGroup */, BdbConsumeGroupSettingEntity> consumeGroupSettingMap;
+ private ConcurrentHashMap<String /* recordKey */, BdbClusterSettingEntity> clusterSettingMap;
private AtomicLong brokerInfoCheckSum = new AtomicLong(System.currentTimeMillis());
private long lastBrokerUpdatedTime = System.currentTimeMillis();
private long serviceStartTime = System.currentTimeMillis();
@@ -98,6 +100,8 @@ public class BrokerConfManager implements Server {
public BrokerConfManager(DefaultBdbStoreService mBdbStoreManagerService) {
this.mBdbStoreManagerService = mBdbStoreManagerService;
this.replicationConfig = mBdbStoreManagerService.getReplicationConfig();
+ this.clusterSettingMap =
+ this.mBdbStoreManagerService.getClusterDefSettingMap();
this.brokerConfStoreMap = this.mBdbStoreManagerService.getBrokerConfigMap();
for (BdbBrokerConfEntity entity : this.brokerConfStoreMap.values()) {
updateBrokerMaps(entity);
@@ -2013,6 +2017,97 @@ public class BrokerConfManager implements Server {
return true;
}
+ // /////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+ /**
+ * Add cluster default setting
+ *
+ * @param bdbEntity the cluster default setting entity will be add
+ * @return true if success otherwise false
+ * @throws Exception
+ */
+ public boolean confAddBdbClusterDefSetting(BdbClusterSettingEntity bdbEntity)
+ throws Exception {
+ validMasterStatus();
+ BdbClusterSettingEntity curEntity =
+ clusterSettingMap.get(bdbEntity.getRecordKey());
+ if (curEntity != null) {
+ throw new Exception(new StringBuilder(512)
+ .append("Duplicate add ClusterSetting info, exist record is: ")
+ .append(curEntity).toString());
+ }
+ boolean putResult =
+ mBdbStoreManagerService.putBdbClusterConfEntity(bdbEntity, true);
+ if (putResult) {
+ clusterSettingMap.put(bdbEntity.getRecordKey(), bdbEntity);
+ logger.info(new StringBuilder(512)
+ .append("[ClusterSetting Success] ")
+ .append(bdbEntity).toString());
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * update cluster default setting
+ *
+ * @param bdbEntity the cluster setting entity will be set
+ * @return true if success otherwise false
+ * @throws Exception
+ */
+ public boolean confUpdBdbClusterSetting(BdbClusterSettingEntity bdbEntity)
+ throws Exception {
+ validMasterStatus();
+ StringBuilder strBuffer = new StringBuilder(512);
+ BdbClusterSettingEntity curDefSettingEntity =
+ clusterSettingMap.get(bdbEntity.getRecordKey());
+ if (curDefSettingEntity == null) {
+ throw new Exception(strBuffer
+ .append("Update ClusterSetting failure, not exist record for record: ")
+ .append(bdbEntity.getRecordKey()).toString());
+ }
+ boolean putResult =
+ mBdbStoreManagerService.putBdbClusterConfEntity(bdbEntity, false);
+ if (putResult) {
+ clusterSettingMap.put(bdbEntity.getRecordKey(), bdbEntity);
+ strBuffer.append("[confUpdBdbClusterSetting Success] record from : ");
+ strBuffer = curDefSettingEntity.toJsonString(strBuffer);
+ strBuffer.append(" to : ");
+ strBuffer = bdbEntity.toJsonString(strBuffer);
+ logger.info(strBuffer.toString());
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Delete cluster default setting
+ *
+ * @param strBuffer the error info string buffer
+ * @return true if success
+ * @throws Exception
+ */
+ public boolean confDeleteBdbClusterSetting(final StringBuilder strBuffer) throws Exception {
+ validMasterStatus();
+ BdbClusterSettingEntity curEntity =
+ this.clusterSettingMap.remove(TServerConstants.TOKEN_DEFAULT_CLUSTER_SETTING);
+ if (curEntity != null) {
+ mBdbStoreManagerService.delBdbClusterConfEntity();
+ strBuffer.append(
+ "[confDeleteBdbClusterSetting Success], deleted cluster setting record :");
+ logger.info(curEntity.toJsonString(strBuffer).toString());
+ strBuffer.delete(0, strBuffer.length());
+ } else {
+ logger.info("[confDeleteBdbClusterSetting Success], not found record");
+ }
+ return true;
+ }
+
+ public BdbClusterSettingEntity getBdbClusterSetting() {
+ return this.clusterSettingMap.get(TServerConstants.TOKEN_DEFAULT_CLUSTER_SETTING);
+ }
+
+
private void validMasterStatus() throws Exception {
if (!isSelfMaster()) {
throw new StandbyException("Please send your request to the master Node.");
[incubator-tubemq] 14/49: [TUBEMQ-453] TubemqSourceFunction class
prints too many logs problem (#350)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 3b6d610035952a9e10cfedee35bd4d5a182ac81e
Author: leno1001 <19...@qq.com>
AuthorDate: Tue Dec 15 22:08:08 2020 +0800
[TUBEMQ-453] TubemqSourceFunction class prints too many logs problem (#350)
Co-authored-by: 曹显乐 <xi...@vivo.com>
---
.../java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java
index c93e057..33c498e 100644
--- a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java
+++ b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java
@@ -264,7 +264,6 @@ public class TubemqSourceFunction<T>
Duration idleTime =
Duration.between(lastConsumeInstant, Instant.now());
if (idleTime.compareTo(maxIdleTime) > 0) {
- LOG.info("Mark this source as temporarily idle.");
ctx.markAsTemporarilyIdle();
}
[incubator-tubemq] 03/49: [TUBEMQ-430]Optimizing the implementation
of HTTP API for broker (#338)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 9598a42a8675657cdf9190d6c75b1a21b1be70e0
Author: gosonzhang <46...@qq.com>
AuthorDate: Fri Dec 4 10:16:15 2020 +0800
[TUBEMQ-430]Optimizing the implementation of HTTP API for broker (#338)
---
.../tubemq/client/config/ConsumerConfig.java | 12 +
.../client/consumer/PullMessageConsumer.java | 7 +
.../org/apache/tubemq/corebase/utils/RegexDef.java | 60 ++++
.../server/broker/web/AbstractWebHandler.java | 83 +++++
.../server/broker/web/BrokerAdminServlet.java | 333 ++++++++++-----------
.../tubemq/server/common/fielddef/CliArgDef.java | 111 +++++++
.../tubemq/server/common/fielddef/WebFieldDef.java | 159 ++++++++++
.../tubemq/server/common/utils/ProcessResult.java | 53 ++++
.../server/common/utils/WebParameterUtils.java | 320 +++++++++++++++++++-
.../tubemq/server/common/webbase/WebFieldType.java | 60 ++++
.../webbase/WebMethodMapper.java} | 12 +-
.../server/master/web/action/screen/Webapi.java | 6 +-
.../master/web/handler/AbstractWebHandler.java | 2 +-
13 files changed, 1037 insertions(+), 181 deletions(-)
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/config/ConsumerConfig.java b/tubemq-client/src/main/java/org/apache/tubemq/client/config/ConsumerConfig.java
index d8b63fb..184da79 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/config/ConsumerConfig.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/config/ConsumerConfig.java
@@ -124,6 +124,18 @@ public class ConsumerConfig extends TubeClientConfig {
return pullConsumeReadyWaitPeriodMs;
}
+ // setPullConsumeReadyWaitPeriodMs() use note:
+ // The value range is [negative value, 0, positive value] and the value directly determines
+ // the behavior of the PullMessageConsumer.GetMessage() function:
+ // 1. if it is set to a negative value, it means that the GetMessage() calling thread will
+ // be blocked forever and will not return until the consumption conditions are met;
+ // 2. if If it is set to 0, it means that the GetMessage() calling thread will only block
+ // the ConsumerConfig.getPullConsumeReadyChkSliceMs() interval when the consumption
+ // conditions are not met and then return;
+ // 3. if it is set to a positive number, it will not meet the current user usage (including
+ // unused partitions or allocated partitions, but these partitions do not meet the usage
+ // conditions), the GetMessage() calling thread will be blocked until the total time of
+ // ConsumerConfig.getPullConsumeReadyWaitPeriodMs expires
public void setPullConsumeReadyWaitPeriodMs(long pullConsumeReadyWaitPeriodMs) {
this.pullConsumeReadyWaitPeriodMs = pullConsumeReadyWaitPeriodMs;
}
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/PullMessageConsumer.java b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/PullMessageConsumer.java
index af5d50f..d9c3baf 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/PullMessageConsumer.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/consumer/PullMessageConsumer.java
@@ -28,6 +28,13 @@ public interface PullMessageConsumer extends MessageConsumer {
PullMessageConsumer subscribe(String topic,
TreeSet<String> filterConds) throws TubeClientException;
+ // getMessage() use note:
+ // This getMessage have a blocking situation: when the current
+ // consumer consumption situation is not satisfied (including
+ // without partitions to consumption, or allocated partitions but
+ // the partitions do not meet the consumption situation),
+ // the call will sleep at intervals of ConsumerConfig.getPullConsumeReadyChkSliceMs(),
+ // until the total time of ConsumerConfig.getPullConsumeReadyWaitPeriodMs
ConsumerResult getMessage() throws TubeClientException;
ConsumerResult confirmConsume(final String confirmContext,
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/RegexDef.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/RegexDef.java
new file mode 100644
index 0000000..842c6a9
--- /dev/null
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/RegexDef.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.corebase.utils;
+
+
+
+public enum RegexDef {
+
+ TMP_FILTER(0, "^[_A-Za-z0-9]+$",
+ "must only contain characters,numbers,and underscores"),
+ TMP_STRING(1, "^[a-zA-Z]\\w+$",
+ "must begin with a letter,can only contain characters,numbers,and underscores"),
+ TMP_NUMBER(2, "^-?[0-9]\\d*$", "must only contain numbers"),
+ TMP_GROUP(3, "^[a-zA-Z][\\w-]+$",
+ "must begin with a letter,can only contain characters,numbers,hyphen,and underscores"),
+ TMP_CONSUMERID(4, "^[_A-Za-z0-9\\.\\-]+$",
+ "must begin with a letter,can only contain characters,numbers,dot,scores,and underscores"),
+ TMP_IPV4ADDRESS(5,
+ "((?:(?:25[0-5]|2[0-4]\\d|((1\\d{2})|([1-9]?\\d)))\\.){3}(?:25[0-5]|2[0-4]\\d|((1\\d{2})|([1-9]?\\d))))",
+ "must matches the IP V4 address regulation");
+
+
+ private final int id;
+ private final String pattern;
+ private final String errMsgTemp;
+
+
+ RegexDef(int id, String pattern, String errMsgTemp) {
+ this.id = id;
+ this.pattern = pattern;
+ this.errMsgTemp = errMsgTemp;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ public String getPattern() {
+ return pattern;
+ }
+
+ public String getErrMsgTemp() {
+ return errMsgTemp;
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
new file mode 100644
index 0000000..b44d88c
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.broker.web;
+
+import static org.apache.tubemq.server.common.webbase.WebMethodMapper.getWebApiRegInfo;
+import static org.apache.tubemq.server.common.webbase.WebMethodMapper.registerWebMethod;
+import java.io.IOException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import org.apache.tubemq.server.broker.TubeBroker;
+import org.apache.tubemq.server.common.webbase.WebMethodMapper.WebApiRegInfo;
+
+
+
+public abstract class AbstractWebHandler extends HttpServlet {
+
+ protected final TubeBroker broker;
+
+ public AbstractWebHandler(TubeBroker broker) {
+ this.broker = broker;
+ }
+
+ @Override
+ protected void doGet(HttpServletRequest req,
+ HttpServletResponse resp) throws IOException {
+ doPost(req, resp);
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req,
+ HttpServletResponse resp) throws IOException {
+ StringBuilder strBuffer = new StringBuilder(1024);
+
+ try {
+ String method = req.getParameter("method");
+ if (method == null) {
+ strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
+ .append("Please take with method parameter! \"}");
+ } else {
+ WebApiRegInfo webApiRegInfo = getWebApiRegInfo(true, method);
+ if (webApiRegInfo == null) {
+ strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
+ .append("Unsupported method ").append(method).append("}");
+ } else {
+ strBuffer = (StringBuilder) webApiRegInfo.method.invoke(webApiRegInfo.webHandler, req);
+ }
+ }
+ } catch (Throwable e) {
+ strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
+ .append("Bad request from server: ")
+ .append(e.getMessage())
+ .append("\"}");
+ }
+ resp.getWriter().write(strBuffer.toString());
+ resp.setCharacterEncoding(req.getCharacterEncoding());
+ resp.setStatus(HttpServletResponse.SC_OK);
+ resp.flushBuffer();
+ }
+
+ public abstract void registerWebApiMethod();
+
+ protected void innRegisterWebMethod(String webMethodName,
+ String clsMethodName) {
+ registerWebMethod(true, webMethodName, clsMethodName, this);
+ }
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index e91ae79..ab43c2e 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -17,18 +17,12 @@
package org.apache.tubemq.server.broker.web;
-import java.io.IOException;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.broker.TubeBroker;
@@ -36,83 +30,44 @@ import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.nodeinfo.ConsumerNodeInfo;
import org.apache.tubemq.server.broker.offset.OffsetService;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
+import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.common.utils.WebParameterUtils;
/***
* Broker's web servlet. Used for admin operation, like query consumer's status etc.
*/
-public class BrokerAdminServlet extends HttpServlet {
- private final TubeBroker broker;
+public class BrokerAdminServlet extends AbstractWebHandler {
- public BrokerAdminServlet(TubeBroker broker) {
- this.broker = broker;
- }
- @Override
- protected void doGet(HttpServletRequest req,
- HttpServletResponse resp) throws ServletException, IOException {
- doPost(req, resp);
+ public BrokerAdminServlet(TubeBroker broker) {
+ super(broker);
+ registerWebApiMethod();
}
@Override
- protected void doPost(HttpServletRequest req,
- HttpServletResponse resp) throws ServletException, IOException {
- StringBuilder sBuilder = new StringBuilder(1024);
- try {
- String method = req.getParameter("method");
- if ("admin_manual_set_current_offset".equals(method)) {
- // manual set offset
- sBuilder = this.adminManualSetCurrentOffSet(req);
- } else if ("admin_query_group_offset".equals(method)) {
- // query consumer group's offset
- sBuilder = this.adminQueryCurrentGroupOffSet(req);
- } else if ("admin_snapshot_message".equals(method)) {
- // query snapshot message
- sBuilder = this.adminQuerySnapshotMessageSet(req);
- } else if ("admin_query_broker_all_consumer_info".equals(method)) {
- // query broker's all consumer info
- sBuilder = this.adminQueryBrokerAllConsumerInfo(req);
- } else if ("admin_query_broker_memstore_info".equals(method)) {
- // get memory store status info
- sBuilder = this.adminGetMemStoreStatisInfo(req);
- } else if ("admin_query_broker_all_store_info".equals(method)) {
- // query broker's all message store info
- sBuilder = this.adminQueryBrokerAllMessageStoreInfo(req);
- } else if ("admin_query_consumer_regmap".equals(method)) {
- Map<String, ConsumerNodeInfo> map =
- broker.getBrokerServiceServer().getConsumerRegisterMap();
- int totalCnt = 0;
- sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",")
- .append(",\"dataSet\":[");
- for (Entry<String, ConsumerNodeInfo> entry : map.entrySet()) {
- if (entry.getKey() == null || entry.getValue() == null) {
- continue;
- }
- if (totalCnt > 0) {
- sBuilder.append(",");
- }
- sBuilder.append("{\"Partition\":\"").append(entry.getKey())
- .append("\",\"Consumer\":\"")
- .append(entry.getValue().getConsumerId())
- .append("\",\"index\":").append(++totalCnt).append("}");
- }
- sBuilder.append("],\"totalCnt\":").append(totalCnt).append("}");
- } else {
- sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
- .append("Invalid request: Unsupported method!")
- .append("\"}");
- }
-
- } catch (Exception e) {
- sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
- .append("Bad request from server: ")
- .append(e.getMessage())
- .append("\"}");
- }
- resp.getWriter().write(sBuilder.toString());
- resp.setCharacterEncoding(req.getCharacterEncoding());
- resp.setStatus(HttpServletResponse.SC_OK);
- resp.flushBuffer();
+ public void registerWebApiMethod() {
+ // query consumer group's offset
+ innRegisterWebMethod("admin_query_group_offset",
+ "adminQueryCurrentGroupOffSet");
+ // query snapshot message
+ innRegisterWebMethod("admin_snapshot_message",
+ "adminQuerySnapshotMessageSet");
+ // query broker's all consumer info
+ innRegisterWebMethod("admin_query_broker_all_consumer_info",
+ "adminQueryBrokerAllConsumerInfo");
+ // get memory store status info
+ innRegisterWebMethod("admin_query_broker_memstore_info",
+ "adminGetMemStoreStatisInfo");
+ // query broker's all message store info
+ innRegisterWebMethod("admin_query_broker_all_store_info",
+ "adminQueryBrokerAllMessageStoreInfo");
+ // query consumer register info
+ innRegisterWebMethod("admin_query_consumer_regmap",
+ "adminQueryConsumerRegisterInfo");
+ // manual set offset
+ innRegisterWebMethod("admin_manual_set_current_offset",
+ "adminManualSetCurrentOffSet");
}
/***
@@ -122,13 +77,16 @@ public class BrokerAdminServlet extends HttpServlet {
* @return
* @throws Exception
*/
- private StringBuilder adminQueryBrokerAllConsumerInfo(HttpServletRequest req) throws Exception {
+ public StringBuilder adminQueryBrokerAllConsumerInfo(HttpServletRequest req) throws Exception {
int index = 0;
StringBuilder sBuilder = new StringBuilder(1024);
- String groupNameInput =
- WebParameterUtils.validGroupParameter("groupName",
- req.getParameter("groupName"),
- TBaseConstants.META_MAX_GROUPNAME_LENGTH, false, null);
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, false, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ Set<String> groupNameSet = (Set<String>) result.retData1;
+
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
Map<String, ConsumerNodeInfo> map =
broker.getBrokerServiceServer().getConsumerRegisterMap();
@@ -139,7 +97,7 @@ public class BrokerAdminServlet extends HttpServlet {
String[] partitionIdArr =
entry.getKey().split(TokenConstants.ATTR_SEP);
String groupName = partitionIdArr[0];
- if (!TStringUtils.isBlank(groupNameInput) && (!groupNameInput.equals(groupName))) {
+ if (!groupNameSet.isEmpty() && !groupNameSet.contains(groupName)) {
continue;
}
String topicName = partitionIdArr[1];
@@ -209,22 +167,23 @@ public class BrokerAdminServlet extends HttpServlet {
* @return
* @throws Exception
*/
- private StringBuilder adminQueryBrokerAllMessageStoreInfo(HttpServletRequest req)
+ public StringBuilder adminQueryBrokerAllMessageStoreInfo(HttpServletRequest req)
throws Exception {
StringBuilder sBuilder = new StringBuilder(1024);
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ Set<String> topicNameSet = (Set<String>) result.retData1;
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
- String topicNameInput =
- WebParameterUtils.validStringParameter("topicName",
- req.getParameter("topicName"),
- TBaseConstants.META_MAX_TOPICNAME_LENGTH, false, null);
Map<String, ConcurrentHashMap<Integer, MessageStore>> messageTopicStores =
broker.getStoreManager().getMessageStores();
int index = 0;
int recordId = 0;
for (Map.Entry<String, ConcurrentHashMap<Integer, MessageStore>> entry : messageTopicStores.entrySet()) {
- if (TStringUtils.isBlank(entry.getKey()) ||
- (TStringUtils.isNotBlank(topicNameInput)
- && !topicNameInput.equals(entry.getKey()))) {
+ if (TStringUtils.isBlank(entry.getKey())
+ || (!topicNameSet.isEmpty() && !topicNameSet.contains(entry.getKey()))) {
continue;
}
if (recordId > 0) {
@@ -276,47 +235,27 @@ public class BrokerAdminServlet extends HttpServlet {
* @return
* @throws Exception
*/
- private StringBuilder adminGetMemStoreStatisInfo(HttpServletRequest req) throws Exception {
+ public StringBuilder adminGetMemStoreStatisInfo(HttpServletRequest req) throws Exception {
StringBuilder sBuilder = new StringBuilder(1024);
- Set<String> batchTopicNames = new HashSet<>();
- String inputTopicName = req.getParameter("topicName");
- if (TStringUtils.isNotBlank(inputTopicName)) {
- inputTopicName = inputTopicName.trim();
- String[] strTopicNames =
- inputTopicName.split(TokenConstants.ARRAY_SEP);
- for (int i = 0; i < strTopicNames.length; i++) {
- if (TStringUtils.isBlank(strTopicNames[i])) {
- continue;
- }
- String topicName = strTopicNames[i].trim();
- if (topicName.length() > TBaseConstants.META_MAX_TOPICNAME_LENGTH) {
- sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
- .append("Invalid parameter: the max length of ")
- .append(topicName).append(" in topicName parameter over ")
- .append(TBaseConstants.META_MAX_TOPICNAME_LENGTH)
- .append(" characters\"}");
- return sBuilder;
- }
- if (!topicName.matches(TBaseConstants.META_TMP_STRING_VALUE)) {
- sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
- .append("Invalid parameter: the value of ").append(topicName)
- .append(" in topicName parameter must begin with a letter,")
- .append(" can only contain characters,numbers,and underscores!\"}");
- return sBuilder;
- }
- batchTopicNames.add(topicName);
- }
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ Set<String> topicNameSet = (Set<String>) result.retData1;
+ result = WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.NEEDREFRESH, false, false);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
}
- boolean requireRefresh =
- WebParameterUtils.validBooleanDataParameter("needRefresh",
- req.getParameter("needRefresh"), false, false);
+ boolean requireRefresh = (boolean) result.retData1;
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"detail\":[");
Map<String, ConcurrentHashMap<Integer, MessageStore>> messageTopicStores =
broker.getStoreManager().getMessageStores();
int recordId = 0, index = 0;
for (Map.Entry<String, ConcurrentHashMap<Integer, MessageStore>> entry : messageTopicStores.entrySet()) {
if (TStringUtils.isBlank(entry.getKey())
- || (!batchTopicNames.isEmpty() && !batchTopicNames.contains(entry.getKey()))) {
+ || (!topicNameSet.isEmpty() && !topicNameSet.contains(entry.getKey()))) {
continue;
}
String topicName = entry.getKey();
@@ -354,25 +293,38 @@ public class BrokerAdminServlet extends HttpServlet {
* @return
* @throws Exception
*/
- private StringBuilder adminManualSetCurrentOffSet(HttpServletRequest req) throws Exception {
+ public StringBuilder adminManualSetCurrentOffSet(HttpServletRequest req) throws Exception {
StringBuilder sBuilder = new StringBuilder(512);
- final String topicName =
- WebParameterUtils.validStringParameter("topicName",
- req.getParameter("topicName"),
- TBaseConstants.META_MAX_TOPICNAME_LENGTH, true, "");
- final String groupName =
- WebParameterUtils.validGroupParameter("groupName",
- req.getParameter("groupName"),
- TBaseConstants.META_MAX_GROUPNAME_LENGTH, true, "");
- final String modifyUser =
- WebParameterUtils.validStringParameter("modifyUser",
- req.getParameter("modifyUser"), 64, true, "");
- int partitionId =
- WebParameterUtils.validIntDataParameter("partitionId",
- req.getParameter("partitionId"), true, -1, 0);
- long manualOffset =
- WebParameterUtils.validLongDataParameter("manualOffset",
- req.getParameter("manualOffset"), true, -1);
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TOPICNAME, true, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ final String topicName = (String) result.retData1;
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.GROUPNAME, true, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ final String groupName = (String) result.retData1;
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ final String modifyUser = (String) result.retData1;
+ result = WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.PARTITIONID, true, -1, 0);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ int partitionId = (Integer) result.retData1;
+ result = WebParameterUtils.getLongParamValue(req,
+ WebFieldDef.MANUALOFFSET, true, -1);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ long manualOffset = (Long) result.retData1;
List<String> topicList = broker.getMetadataManager().getTopics();
if (!topicList.contains(topicName)) {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
@@ -430,28 +382,39 @@ public class BrokerAdminServlet extends HttpServlet {
* @return
* @throws Exception
*/
- private StringBuilder adminQuerySnapshotMessageSet(HttpServletRequest req) throws Exception {
+ public StringBuilder adminQuerySnapshotMessageSet(HttpServletRequest req) throws Exception {
StringBuilder sBuilder = new StringBuilder(1024);
- final String topicName =
- WebParameterUtils.validStringParameter("topicName",
- req.getParameter("topicName"),
- TBaseConstants.META_MAX_TOPICNAME_LENGTH, true, "");
- final int partitionId =
- WebParameterUtils.validIntDataParameter("partitionId",
- req.getParameter("partitionId"), false, -1, 0);
- int msgCount =
- WebParameterUtils.validIntDataParameter("msgCount",
- req.getParameter("msgCount"), false, 3, 3);
- msgCount = msgCount < 1 ? 1 : msgCount;
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TOPICNAME, true, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ final String topicName = (String) result.retData1;
+ result = WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.PARTITIONID, true, -1, 0);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ int partitionId = (Integer) result.retData1;
+ result = WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.MSGCOUNT, false, 3, 3);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ int msgCount = (Integer) result.retData1;
+ msgCount = Math.max(msgCount, 1);
if (msgCount > 50) {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Over max allowed msgCount value, allowed count is 50!")
.append("\"}");
return sBuilder;
}
- Set<String> filterCondStrSet =
- WebParameterUtils.checkAndGetFilterCondSet(req.getParameter("filterConds"),
- false, true, sBuilder);
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.FILTERCONDS, false, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ Set<String> filterCondStrSet = (Set<String>) result.retData1;
sBuilder = broker.getBrokerServiceServer()
.getMessageSnapshot(topicName, partitionId, msgCount, filterCondStrSet, sBuilder);
return sBuilder;
@@ -464,20 +427,34 @@ public class BrokerAdminServlet extends HttpServlet {
* @return
* @throws Exception
*/
- private StringBuilder adminQueryCurrentGroupOffSet(HttpServletRequest req)
+ public StringBuilder adminQueryCurrentGroupOffSet(HttpServletRequest req)
throws Exception {
StringBuilder sBuilder = new StringBuilder(1024);
- String topicName =
- WebParameterUtils.validStringParameter("topicName",
- req.getParameter("topicName"),
- TBaseConstants.META_MAX_TOPICNAME_LENGTH, true, "");
- String groupName =
- WebParameterUtils.validGroupParameter("groupName",
- req.getParameter("groupName"),
- TBaseConstants.META_MAX_GROUPNAME_LENGTH, true, "");
- int partitionId =
- WebParameterUtils.validIntDataParameter("partitionId",
- req.getParameter("partitionId"), true, -1, 0);
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TOPICNAME, true, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ final String topicName = (String) result.retData1;
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.GROUPNAME, true, null);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ final String groupName = (String) result.retData1;
+ result = WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.PARTITIONID, true, -1, 0);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ int partitionId = (Integer) result.retData1;
+
+ result = WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.REQUIREREALOFFSET, false, false);
+ if (!result.success) {
+ return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ }
+ boolean requireRealOffset = (Boolean) result.retData1;
List<String> topicList = broker.getMetadataManager().getTopics();
if (!topicList.contains(topicName)) {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
@@ -499,9 +476,6 @@ public class BrokerAdminServlet extends HttpServlet {
.append("\"}");
return sBuilder;
}
- boolean requireRealOffset =
- WebParameterUtils.validBooleanDataParameter("requireRealOffset",
- req.getParameter("requireRealOffset"), false, false);
long tmpOffset = offsetService.getTmpOffset(groupName, topicName, partitionId);
long minDataOffset = store.getDataMinOffset();
long maxDataOffset = store.getDataMaxOffset();
@@ -538,5 +512,28 @@ public class BrokerAdminServlet extends HttpServlet {
return sBuilder;
}
+ public StringBuilder adminQueryConsumerRegisterInfo(HttpServletRequest req) {
+ StringBuilder sBuilder = new StringBuilder(1024);
+ Map<String, ConsumerNodeInfo> map =
+ broker.getBrokerServiceServer().getConsumerRegisterMap();
+ int totalCnt = 0;
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",")
+ .append(",\"dataSet\":[");
+ for (Entry<String, ConsumerNodeInfo> entry : map.entrySet()) {
+ if (entry.getKey() == null || entry.getValue() == null) {
+ continue;
+ }
+ if (totalCnt > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("{\"Partition\":\"").append(entry.getKey())
+ .append("\",\"Consumer\":\"")
+ .append(entry.getValue().getConsumerId())
+ .append("\",\"index\":").append(++totalCnt).append("}");
+ }
+ sBuilder.append("],\"totalCnt\":").append(totalCnt).append("}");
+ return sBuilder;
+ }
+
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
new file mode 100644
index 0000000..b2d9327
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.common.fielddef;
+
+
+
+public enum CliArgDef {
+
+ // Note: Due to compatibility considerations,
+ // the defined fields in the scheme are forbidden to be modified,
+ // only new fields can be added
+
+ HELP("h", "help", "Print usage information."),
+ VERSION("v", "version", "Display TubeMQ version."),
+ MASTERSERVER("master-servers", "master-servers",
+ "String: format is master1_ip:port[,master2_ip:port]",
+ "The master address(es) to connect to."),
+ MASTERURL("master-url", "master-url",
+ "String: format is http://master_ip:master_webport/",
+ "Master Service URL to which to connect.(default: http://localhost:8080/)"),
+ BROKERURL("broker-url", "broker-url",
+ "String: format is http://broker_ip:broker_webport/",
+ "Broker Service URL to which to connect.(default: http://localhost:8081/)"),
+ MESSAGES("messages", "messages",
+ "Long: count",
+ "The number of messages to send or consume, If not set, production or consumption is continual."),
+ MSGDATASIZE("msg-data-size", "message-data-size",
+ "Int: message size",
+ "message's data size in bytes. Note that you must provide exactly"
+ + " one of --msg-data-size or --payload-file."),
+ PAYLOADFILE("payload-file", "payload-file",
+ "String: payload file path",
+ "file to read the message payloads from. This works only for"
+ + " UTF-8 encoded text files. Payloads will be read from this"
+ + " file and a payload will be randomly selected when sending"
+ + " messages. Note that you must provide exactly one"
+ + " of --msg-data-size or --payload-file."),
+ PAYLOADDELIM("payload-delimiter", "payload-delimiter",
+ "String: payload data's delimiter",
+ "provides delimiter to be used when --payload-file is provided."
+ + " Defaults to new line. Note that this parameter will be"
+ + " ignored if --payload-file is not provided. (default: \\n)"),
+ PRDTOPIC("topic", "topicName",
+ "String: topic, format is topic_1[,topic_2[:filterCond_2.1[;filterCond_2.2]]]",
+ "The topic(s) to produce messages to."),
+ CNSTOPIC("topic", "topicName",
+ "String: topic, format is topic_1[[:filterCond_1.1[;filterCond_1.2]][,topic_2]]",
+ "The topic(s) to consume on."),
+ RPCTIMEOUT("timeout", "timeout",
+ "Long: milliseconds",
+ "The maximum duration between request and response in milliseconds. (default: 10000)"),
+ GROUP("group", "groupName",
+ "String: consumer group",
+ "The consumer group name of the consumer."),
+ CLIENTCOUNT("client-num", "client-num",
+ "Int: client count",
+ "Number of consumers to started."),
+ PULLMODEL("pull-model", "pull-model",
+ "Pull consumption model."),
+ PUSHMODEL("push-model", "push-model",
+ "Push consumption model."),
+ FETCHTHREADS("num-fetch-threads", "num-fetch-threads",
+ "Integer: count",
+ "Number of fetch threads, default: num of cpu count."),
+ FROMLATEST("from-latest", "from-latest",
+ "Start to consume from the latest message present in the log."),
+ FROMBEGINNING("from-beginning", "from-beginning",
+ "If the consumer does not already have an established offset to consume from,"
+ + " start with the earliest message present in the log rather than the latest message."),
+ OUTPUTINTERVAL("output-interval", "output-interval",
+ "Integer: interval_ms",
+ "Interval in milliseconds at which to print progress info. (default: 5000)");
+
+
+ CliArgDef(String opt, String longOpt, String optDesc) {
+ this(opt, longOpt, false, "", optDesc);
+ }
+
+ CliArgDef(String opt, String longOpt, String argDesc, String optDesc) {
+ this(opt, longOpt, true, argDesc, optDesc);
+ }
+
+ CliArgDef(String opt, String longOpt, boolean hasArg, String argDesc, String optDesc) {
+ this.opt = opt;
+ this.longOpt = longOpt;
+ this.hasArg = hasArg;
+ this.argDesc = argDesc;
+ this.optDesc = optDesc;
+ }
+
+ public final String opt;
+ public final String longOpt;
+ public final boolean hasArg;
+ public final String argDesc;
+ public final String optDesc;
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
new file mode 100644
index 0000000..1025ba0
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.common.fielddef;
+
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.TokenConstants;
+import org.apache.tubemq.corebase.utils.RegexDef;
+import org.apache.tubemq.server.common.TServerConstants;
+import org.apache.tubemq.server.common.webbase.WebFieldType;
+
+
+public enum WebFieldDef {
+
+ // Note: Due to compatibility considerations,
+ // the defined fields in the scheme are forbidden to be modified,
+ // only new fields can be added
+
+ TOPICNAME(0, "topicName", "topic", WebFieldType.STRING,
+ "Topic name", TBaseConstants.META_MAX_TOPICNAME_LENGTH,
+ RegexDef.TMP_STRING),
+ GROUPNAME(1, "groupName", "group", WebFieldType.STRING,
+ "Group name", TBaseConstants.META_MAX_GROUPNAME_LENGTH,
+ RegexDef.TMP_GROUP),
+ PARTITIONID(2, "partitionId", "pid", WebFieldType.INT,
+ "Partition id", RegexDef.TMP_NUMBER),
+ CREATEUSER(3, "createUser", "cur", WebFieldType.STRING,
+ "Record creator", TBaseConstants.META_MAX_USERNAME_LENGTH,
+ RegexDef.TMP_STRING),
+ MODIFYUSER(4, "modifyUser", "mur", WebFieldType.STRING,
+ "Record modifier", TBaseConstants.META_MAX_USERNAME_LENGTH,
+ RegexDef.TMP_STRING),
+ MANUALOFFSET(5, "manualOffset", "offset", WebFieldType.LONG,
+ "Reset offset value", RegexDef.TMP_NUMBER),
+ MSGCOUNT(6, "msgCount", "cnt", WebFieldType.INT,
+ "Number of returned messages", RegexDef.TMP_NUMBER),
+ FILTERCONDS(7, "filterConds", "flts", WebFieldType.COMPSTRING,
+ "Filter condition items", TBaseConstants.CFG_FLT_MAX_FILTER_ITEM_LENGTH,
+ TBaseConstants.CFG_FLT_MAX_FILTER_ITEM_COUNT, RegexDef.TMP_FILTER),
+ REQUIREREALOFFSET(8, "requireRealOffset", "dko", WebFieldType.BOOLEAN,
+ "Require return disk offset details"),
+ NEEDREFRESH(9, "needRefresh", "nrf", WebFieldType.BOOLEAN,
+ "Require refresh data"),
+ COMPSGROUPNAME(10, "groupName", "group", WebFieldType.COMPSTRING,
+ "Group name", TBaseConstants.META_MAX_GROUPNAME_LENGTH,
+ RegexDef.TMP_GROUP),
+ COMPSTOPICNAME(11, "topicName", "topic", WebFieldType.COMPSTRING,
+ "Topic name", TBaseConstants.META_MAX_TOPICNAME_LENGTH,
+ RegexDef.TMP_STRING),
+ COMPSPARTITIONID(12, "partitionId", "pid", WebFieldType.COMPINT,
+ "Partition id", RegexDef.TMP_NUMBER);
+
+
+
+
+ public final int id;
+ public final String name;
+ public final String shortName;
+ public final WebFieldType type;
+ public final String desc;
+ public final boolean compVal;
+ public final String splitToken;
+ public final int itemMaxCnt;
+ public final int valMaxLen;
+ public final boolean regexCheck;
+ public final RegexDef regexDef;
+
+
+ WebFieldDef(int id, String name, String shortName, WebFieldType type, String desc) {
+ this(id, name, shortName, type, desc, TBaseConstants.META_VALUE_UNDEFINED,
+ TBaseConstants.META_VALUE_UNDEFINED, false, null);
+ }
+
+ WebFieldDef(int id, String name, String shortName, WebFieldType type,
+ String desc, RegexDef regexDef) {
+ this(id, name, shortName, type, desc,
+ TBaseConstants.META_VALUE_UNDEFINED, regexDef);
+ }
+
+ WebFieldDef(int id, String name, String shortName, WebFieldType type,
+ String desc, int valMaxLen, RegexDef regexDef) {
+ this(id, name, shortName, type, desc, valMaxLen,
+ TServerConstants.CFG_BATCH_RECORD_OPERATE_MAX_COUNT,
+ true, regexDef);
+ }
+
+ WebFieldDef(int id, String name, String shortName, WebFieldType type,
+ String desc, int valMaxLen, int itemMaxCnt, RegexDef regexDef) {
+ this(id, name, shortName, type, desc, valMaxLen,
+ itemMaxCnt, true, regexDef);
+ }
+
+ WebFieldDef(int id, String name, String shortName, WebFieldType type,
+ String desc, int valMaxLen, int itemMaxCnt,
+ boolean regexChk, RegexDef regexDef) {
+ this.id = id;
+ this.name = name;
+ this.shortName = shortName;
+ this.type = type;
+ this.desc = desc;
+ if (isCompFieldType()) {
+ this.compVal = true;
+ this.splitToken = TokenConstants.ARRAY_SEP;
+ this.itemMaxCnt = itemMaxCnt;
+ } else {
+ this.compVal = false;
+ this.splitToken = "";
+ this.itemMaxCnt = TBaseConstants.META_VALUE_UNDEFINED;
+ }
+ this.valMaxLen = valMaxLen;
+ this.regexCheck = regexChk;
+ this.regexDef = regexDef;
+ }
+
+ public boolean isCompFieldType() {
+ return (this.type == WebFieldType.COMPINT
+ || this.type == WebFieldType.COMPSTRING);
+ }
+
+ private static final WebFieldDef[] WEB_FIELD_DEFS;
+ private static final int MIN_FIELD_ID = 0;
+ public static final int MAX_FIELD_ID;
+
+ static {
+ int maxId = -1;
+ for (WebFieldDef fieldDef : WebFieldDef.values()) {
+ maxId = Math.max(maxId, fieldDef.id);
+ }
+ WebFieldDef[] idToType = new WebFieldDef[maxId + 1];
+ for (WebFieldDef fieldDef : WebFieldDef.values()) {
+ idToType[fieldDef.id] = fieldDef;
+ }
+ WEB_FIELD_DEFS = idToType;
+ MAX_FIELD_ID = maxId;
+ }
+
+ public static WebFieldDef valueOf(int fieldId) {
+ if (fieldId >= MIN_FIELD_ID && fieldId <= MAX_FIELD_ID) {
+ return WEB_FIELD_DEFS[fieldId];
+ }
+ throw new IllegalArgumentException(
+ String.format("Unexpected WebFieldDef id `%s`, it should be between `%s` " +
+ "and `%s` (inclusive)", fieldId, MIN_FIELD_ID, MAX_FIELD_ID));
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
new file mode 100644
index 0000000..3688b1c
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/ProcessResult.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.common.utils;
+
+import org.apache.tubemq.corebase.TErrCodeConstants;
+
+public class ProcessResult {
+ public boolean success = true;
+ public int errCode = TErrCodeConstants.SUCCESS;
+ public String errInfo = "";
+ public Object retData1 = null;
+
+ public ProcessResult() {
+
+ }
+
+ public ProcessResult(Object retData) {
+ this.success = true;
+ this.retData1 = retData;
+ }
+
+ public ProcessResult(int errCode, String errInfo) {
+ this.success = false;
+ this.errCode = errCode;
+ this.errInfo = errInfo;
+ }
+
+ public void setFailResult(int errCode, final String errMsg) {
+ this.success = false;
+ this.errCode = errCode;
+ this.errInfo = errMsg;
+ }
+
+ public void setSuccResult(Object retData) {
+ this.success = true;
+ this.retData1 = retData;
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index f4ce5bf..b7d5d41 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -30,18 +30,21 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
+import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.TStatusConstants;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
import org.apache.tubemq.server.master.TMaster;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerConfManager;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerSyncStatusInfo;
+
public class WebParameterUtils {
private static final List<String> allowedDelUnits = Arrays.asList("s", "m", "h");
@@ -153,6 +156,24 @@ public class WebParameterUtils {
/**
* Parse the parameter value from an object value to string value
*
+ * @param req http servlet request
+ * @param paramName the parameter name
+ * @param paramMaxLen the max length of string to return
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defaultValue a default value returned if failed to parse value from the given object
+ * @return a string value of parameter
+ * @throws Exception if failed to parse the object
+ */
+ public static String validStringParameter(HttpServletRequest req, String paramName,
+ int paramMaxLen, boolean required,
+ String defaultValue) throws Exception {
+ return validStringParameter(paramName,
+ req.getParameter(paramName), paramMaxLen, required, defaultValue);
+ }
+
+ /**
+ * Parse the parameter value from an object value to string value
+ *
* @param paramName the parameter name
* @param paramValue the parameter value which is an object for parsing
* @param paramMaxLen the max length of string to return
@@ -161,9 +182,11 @@ public class WebParameterUtils {
* @return a string value of parameter
* @throws Exception if failed to parse the object
*/
- public static String validStringParameter(String paramName, String paramValue, int paramMaxLen,
- boolean required, String defaultValue) throws Exception {
- String tmpParamValue = checkParamCommonRequires(paramName, paramValue, required);
+ public static String validStringParameter(String paramName, String paramValue,
+ int paramMaxLen, boolean required,
+ String defaultValue) throws Exception {
+ String tmpParamValue =
+ checkParamCommonRequires(paramName, paramValue, required);
if (TStringUtils.isBlank(tmpParamValue)) {
return defaultValue;
}
@@ -214,6 +237,297 @@ public class WebParameterUtils {
return tmpParamValue;
}
+ public static StringBuilder buildFailResult(StringBuilder strBuffer, String errMsg) {
+ return strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
+ .append(errMsg).append("\"}");
+ }
+
+ /**
+ * Parse the parameter value from an object value to a long value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if failed to parse value from the given object
+ * @return valid result for the parameter value
+ */
+ public static ProcessResult getLongParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ long defValue) {
+ ProcessResult procResult =
+ getStringParamValue(req, fieldDef, required, null);
+ if (!procResult.success) {
+ return procResult;
+ }
+ String paramValue = (String) procResult.retData1;
+ if (paramValue == null) {
+ procResult.setSuccResult(defValue);
+ return procResult;
+ }
+ try {
+ long paramIntVal = Long.parseLong(paramValue);
+ procResult.setSuccResult(paramIntVal);
+ } catch (Throwable e) {
+ procResult.setFailResult(400,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name).append(" parse error: ")
+ .append(e.getMessage()).toString());
+ }
+ return procResult;
+ }
+
+ /**
+ * Parse the parameter value from an object value to a integer value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if failed to parse value from the given object
+ * @param minValue min value required
+ * @return valid result for the parameter value
+ */
+ public static ProcessResult getIntParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ int defValue,
+ int minValue) {
+ ProcessResult procResult =
+ getStringParamValue(req, fieldDef, required, null);
+ if (!procResult.success) {
+ return procResult;
+ }
+ if (fieldDef.isCompFieldType()) {
+ Set<Integer> tgtValueSet = new HashSet<Integer>();
+ Set<String> valItemSet = (Set<String>) procResult.retData1;
+ if (valItemSet.isEmpty()) {
+ tgtValueSet.add(defValue);
+ procResult.setSuccResult(tgtValueSet);
+ return procResult;
+ }
+ ProcessResult procRet = new ProcessResult();
+ for (String itemVal : valItemSet) {
+ if (!checkIntValueNorms(procRet, fieldDef, itemVal, minValue)) {
+ return procRet;
+ }
+ tgtValueSet.add((Integer) procRet.retData1);
+ }
+ procResult.setSuccResult(tgtValueSet);
+ } else {
+ String paramValue = (String) procResult.retData1;
+ if (paramValue == null) {
+ procResult.setSuccResult(defValue);
+ return procResult;
+ }
+ checkIntValueNorms(procResult, fieldDef, paramValue, minValue);
+ }
+ return procResult;
+ }
+
+ /**
+ * Parse the parameter value from an object value to a boolean value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if failed to parse value from the given object
+ * @return valid result for the parameter value
+ */
+ public static ProcessResult getBooleanParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ boolean defValue) {
+ ProcessResult procResult =
+ getStringParamValue(req, fieldDef, required, null);
+ if (!procResult.success) {
+ return procResult;
+ }
+ String paramValue = (String) procResult.retData1;
+ if (paramValue == null) {
+ procResult.setSuccResult(defValue);
+ return procResult;
+ }
+ procResult.setSuccResult(Boolean.parseBoolean(paramValue));
+ return procResult;
+ }
+
+ /**
+ * Parse the parameter value from an object value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if failed to parse value from the given object
+ * @return valid result for the parameter value
+ */
+ public static ProcessResult getStringParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ String defValue) {
+ ProcessResult procResult = new ProcessResult();
+ // get parameter value
+ String paramValue = req.getParameter(fieldDef.name);
+ if (paramValue == null) {
+ paramValue = req.getParameter(fieldDef.shortName);
+ }
+ if (TStringUtils.isNotBlank(paramValue)) {
+ // Cleanup value extra characters
+ paramValue = escDoubleQuotes(paramValue.trim());
+ }
+ // Check if the parameter exists
+ if (TStringUtils.isBlank(paramValue)) {
+ if (required) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append(" is missing or value is null or blank!").toString());
+ } else {
+ procStringDefValue(procResult, fieldDef.isCompFieldType(), defValue);
+ }
+ return procResult;
+ }
+ // check if value is norm;
+ if (fieldDef.isCompFieldType()) {
+ // split original value to items
+ Set<String> valItemSet = new HashSet<>();
+ String[] strParamValueItems = paramValue.split(fieldDef.splitToken);
+ for (String strParamValueItem : strParamValueItems) {
+ if (TStringUtils.isBlank(strParamValueItem)) {
+ continue;
+ }
+ if (!checkStrValueNorms(procResult, fieldDef, strParamValueItem)) {
+ return procResult;
+ }
+ valItemSet.add((String) procResult.retData1);
+ }
+ // check if is empty result
+ if (valItemSet.isEmpty()) {
+ if (required) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append(" is missing or value is null or blank!").toString());
+ } else {
+ procStringDefValue(procResult, fieldDef.isCompFieldType(), defValue);
+ }
+ return procResult;
+ }
+ // check max item count
+ if (fieldDef.itemMaxCnt != TBaseConstants.META_VALUE_UNDEFINED) {
+ if (valItemSet.size() > fieldDef.itemMaxCnt) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append("'s item count over max allowed count (")
+ .append(fieldDef.itemMaxCnt).append(")!").toString());
+ }
+ }
+ procResult.setSuccResult(valItemSet);
+ } else {
+ if (!checkStrValueNorms(procResult, fieldDef, paramValue)) {
+ return procResult;
+ }
+ procResult.setSuccResult(paramValue);
+ }
+ return procResult;
+ }
+
+ /**
+ * process string default value
+ *
+ * @param procResult process result
+ * @param isCompFieldType the parameter if compound field type
+ * @param defValue the parameter default value
+ * @return process result for default value of parameter
+ */
+ private static ProcessResult procStringDefValue(ProcessResult procResult,
+ boolean isCompFieldType,
+ String defValue) {
+ if (isCompFieldType) {
+ Set<String> valItemSet = new HashSet<>();
+ if (TStringUtils.isNotBlank(defValue)) {
+ valItemSet.add(defValue);
+ }
+ procResult.setSuccResult(valItemSet);
+ } else {
+ procResult.setSuccResult(defValue);
+ }
+ return procResult;
+ }
+
+ /**
+ * Parse the parameter string value by regex define
+ *
+ * @param procResult process result
+ * @param fieldDef the parameter field definition
+ * @param paramVal the parameter value
+ * @return check result for string value of parameter
+ */
+ private static boolean checkStrValueNorms(ProcessResult procResult,
+ WebFieldDef fieldDef,
+ String paramVal) {
+ paramVal = paramVal.trim();
+ if (TStringUtils.isBlank(paramVal)) {
+ procResult.setSuccResult(null);
+ return true;
+ }
+ // check value's max length
+ if (fieldDef.valMaxLen != TBaseConstants.META_VALUE_UNDEFINED) {
+ if (paramVal.length() > fieldDef.valMaxLen) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("over max length for ")
+ .append(fieldDef.name).append(", only allow ")
+ .append(fieldDef.valMaxLen).append(" length").toString());
+ return false;
+ }
+ }
+ // check value's pattern
+ if (fieldDef.regexCheck) {
+ if (!paramVal.matches(fieldDef.regexDef.getPattern())) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("illegal value for ")
+ .append(fieldDef.name).append(", value ")
+ .append(fieldDef.regexDef.getErrMsgTemp()).toString());
+ return false;
+ }
+ }
+ procResult.setSuccResult(paramVal);
+ return true;
+ }
+
+ /**
+ * Parse the parameter string value by regex define
+ *
+ * @param procResult process result
+ * @param fieldDef the parameter field definition
+ * @param paramValue the parameter value
+ * param minValue the parameter min value
+ * @return check result for string value of parameter
+ */
+ private static boolean checkIntValueNorms(ProcessResult procResult,
+ WebFieldDef fieldDef,
+ String paramValue,
+ int minValue) {
+ try {
+ int paramIntVal = Integer.parseInt(paramValue);
+ if (paramIntVal < minValue) {
+ procResult.setFailResult(400,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name).append(" value must >= ")
+ .append(minValue).toString());
+ return false;
+ }
+ procResult.setSuccResult(paramIntVal);
+ } catch (Throwable e) {
+ procResult.setFailResult(400,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name).append(" parse error: ")
+ .append(e.getMessage()).toString());
+ return false;
+ }
+ return true;
+ }
+
/**
* Parse the parameter value from an object value to ip address of string value
*
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
new file mode 100644
index 0000000..b83a966
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.common.webbase;
+
+
+
+public enum WebFieldType {
+
+ UNKNOWN(-1, "Unknown field type"),
+ STRING(1, "String"),
+ INT(2, "int"),
+ LONG(3, "long"),
+ BOOLEAN(4, "Boolean"),
+ DATE(5, "Date"),
+ COMPSTRING(6, "Compound string"),
+ COMPINT(7, "Compound integer");
+
+
+ public int value;
+ public String desc;
+
+ WebFieldType(int value, String desc) {
+ this.value = value;
+ this.desc = desc;
+ }
+
+ public static WebFieldType valueOf(int value) {
+ for (WebFieldType fieldType : WebFieldType.values()) {
+ if (fieldType.getValue() == value) {
+ return fieldType;
+ }
+ }
+
+ return UNKNOWN;
+ }
+
+ public int getValue() {
+ return value;
+ }
+
+ public String getDesc(){
+ return desc;
+ }
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebApiMapper.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
similarity index 87%
rename from tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebApiMapper.java
rename to tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
index a71ba03..a856014 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebApiMapper.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.tubemq.server.master.web.handler;
+package org.apache.tubemq.server.common.webbase;
import java.lang.reflect.Method;
import java.util.HashMap;
@@ -24,10 +24,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class WebApiMapper {
+public class WebMethodMapper {
// log printer
private static final Logger logger =
- LoggerFactory.getLogger(WebApiMapper.class);
+ LoggerFactory.getLogger(WebMethodMapper.class);
// The query methods map
public static final Map<String, WebApiRegInfo> WEB_QRY_METHOD_MAP =
new HashMap<>();
@@ -47,7 +47,7 @@ public class WebApiMapper {
public static void registerWebMethod(boolean isQryApi,
String webMethodName,
String clsMethodName,
- AbstractWebHandler webHandler) {
+ Object webHandler) {
Method[] methods = webHandler.getClass().getMethods();
for (Method item : methods) {
if (item.getName().equals(clsMethodName)) {
@@ -71,10 +71,10 @@ public class WebApiMapper {
public static class WebApiRegInfo {
public Method method;
- public AbstractWebHandler webHandler;
+ public Object webHandler;
public WebApiRegInfo(Method method,
- AbstractWebHandler webHandler) {
+ Object webHandler) {
this.method = method;
this.webHandler = webHandler;
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
index 326f690..5d4de04 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
@@ -17,7 +17,7 @@
package org.apache.tubemq.server.master.web.action.screen;
-import static org.apache.tubemq.server.master.web.handler.WebApiMapper.getWebApiRegInfo;
+import static org.apache.tubemq.server.common.webbase.WebMethodMapper.getWebApiRegInfo;
import java.util.Arrays;
import java.util.List;
@@ -25,13 +25,13 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corerpc.exception.StandbyException;
+import org.apache.tubemq.server.common.webbase.WebMethodMapper;
import org.apache.tubemq.server.master.TMaster;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerConfManager;
import org.apache.tubemq.server.master.web.handler.AbstractWebHandler;
import org.apache.tubemq.server.master.web.handler.WebAdminFlowRuleHandler;
import org.apache.tubemq.server.master.web.handler.WebAdminGroupCtrlHandler;
import org.apache.tubemq.server.master.web.handler.WebAdminTopicAuthHandler;
-import org.apache.tubemq.server.master.web.handler.WebApiMapper;
import org.apache.tubemq.server.master.web.handler.WebBrokerDefConfHandler;
import org.apache.tubemq.server.master.web.handler.WebBrokerTopicConfHandler;
import org.apache.tubemq.server.master.web.handler.WebMasterInfoHandler;
@@ -102,7 +102,7 @@ public class Webapi implements Action {
"DesignatedPrimary happened...please check if the other member is down");
}
}
- WebApiMapper.WebApiRegInfo webApiRegInfo = getWebApiRegInfo(isQuery, method);
+ WebMethodMapper.WebApiRegInfo webApiRegInfo = getWebApiRegInfo(isQuery, method);
if (webApiRegInfo == null) {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"Unsupported method: ")
.append(method).append("}");
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java
index 09b11eb..1b1bfdc 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/AbstractWebHandler.java
@@ -17,7 +17,7 @@
package org.apache.tubemq.server.master.web.handler;
-import static org.apache.tubemq.server.master.web.handler.WebApiMapper.registerWebMethod;
+import static org.apache.tubemq.server.common.webbase.WebMethodMapper.registerWebMethod;
import org.apache.tubemq.server.master.TMaster;
import org.apache.tubemq.server.master.nodemanage.nodebroker.BrokerConfManager;
[incubator-tubemq] 43/49: [TUBEMQ-517] Add 0.8.0 version release
modification to CHANGES.md (#399)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 01307489d410679288cd0cbae2ab4fafbde60a92
Author: Yuanbo Liu <yu...@apache.org>
AuthorDate: Mon Jan 18 17:22:26 2021 +0800
[TUBEMQ-517] Add 0.8.0 version release modification to CHANGES.md (#399)
* [TUBEMQ-517] Add 0.8.0 version release modification to CHANGES.md
---
CHANGES.md | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 67 insertions(+)
diff --git a/CHANGES.md b/CHANGES.md
index 33bfc4d..977b738 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -21,6 +21,73 @@
# of release header or remove the below marker. This file is generated.
# DO NOT REMOVE THIS MARKER; FOR INTERPOLATING CHANGES!-->
+## Release 0.8.0-incubating - Released (as of 2021-01-18)
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority |
+| :---- | :------- | :------- |
+| [TUBEMQ-430](https://issues.apache.org/jira/browse/TUBEMQ-430) | Optimizing the implementation of HTTP API for broke | Major |
+| [TUBEMQ-433](https://issues.apache.org/jira/browse/TUBEMQ-433) | add tubemq perf-consumer/producer scripts | Major |
+| [TUBEMQ-440](https://issues.apache.org/jira/browse/TUBEMQ-440) | Add feature package tube-manager to zip | Major |
+| [TUBEMQ-442](https://issues.apache.org/jira/browse/TUBEMQ-442) | Modifying the jvm parameters when the broker starts does not take effect | Major |
+| [TUBEMQ-444](https://issues.apache.org/jira/browse/TUBEMQ-444) | Add consume and produce Cli commands | Major |
+| [TUBEMQ-445](https://issues.apache.org/jira/browse/TUBEMQ-445) | Adjust the status check default sleep interval of pullConsumeReadyChkSliceMs | Major |
+| [TUBEMQ-447](https://issues.apache.org/jira/browse/TUBEMQ-447) | Add Broker-Admin Cli | Major |
+| [TUBEMQ-449](https://issues.apache.org/jira/browse/TUBEMQ-449) | Adjust Example implementation | Major |
+| [TUBEMQ-450](https://issues.apache.org/jira/browse/TUBEMQ-450) | TubeClientException: Generate producer id failed | Major |
+| [TUBEMQ-451](https://issues.apache.org/jira/browse/TUBEMQ-451) | Replace ConsumeTupleInfo with Tuple2 | Major |
+| [TUBEMQ-463](https://issues.apache.org/jira/browse/TUBEMQ-463) | Adjust Master rebalance process implementation | Major |
+| [TUBEMQ-470](https://issues.apache.org/jira/browse/TUBEMQ-470) | Add query API of TopicName and BrokerId collection | Major |
+| [TUBEMQ-472](https://issues.apache.org/jira/browse/TUBEMQ-472) | Adjust Broker's AbstractWebHandler class implementation | Major |
+| [TUBEMQ-475](https://issues.apache.org/jira/browse/TUBEMQ-475) | add the offset clone api of the consume group | Major |
+| [TUBEMQ-482](https://issues.apache.org/jira/browse/TUBEMQ-482) | Add offset query api | Major |
+| [TUBEMQ-484](https://issues.apache.org/jira/browse/TUBEMQ-484) | Add query API for topic publication information | Major |
+| [TUBEMQ-485](https://issues.apache.org/jira/browse/TUBEMQ-485) | Add the batch setting API of consume group offset | Major |
+| [TUBEMQ-486](https://issues.apache.org/jira/browse/TUBEMQ-486) | Add the delete API of consumer group offset | Major |
+| [TUBEMQ-499](https://issues.apache.org/jira/browse/TUBEMQ-499) | Add configure store | Major |
+| [TUBEMQ-500](https://issues.apache.org/jira/browse/TUBEMQ-500) | Add setting operate API | Major |
+| [TUBEMQ-501](https://issues.apache.org/jira/browse/TUBEMQ-501) | Adjust max message size check logic | Major |
+| [TUBEMQ-502](https://issues.apache.org/jira/browse/TUBEMQ-502) | Add setting API interface document | Major |
+| [TUBEMQ-504](https://issues.apache.org/jira/browse/TUBEMQ-504) | Adjust the WebMethodMapper class interfaces | Major |
+| [TUBEMQ-505](https://issues.apache.org/jira/browse/TUBEMQ-505) | Remove the "WIP" label of the DISCLAIMER file | Major |
+| [TUBEMQ-508](https://issues.apache.org/jira/browse/TUBEMQ-508) | Optimize Broker's PB parameter check processing logic | Major |
+| [TUBEMQ-509](https://issues.apache.org/jira/browse/TUBEMQ-509) | Adjust the packet length check when data is loaded | Major |
+| [TUBEMQ-511](https://issues.apache.org/jira/browse/TUBEMQ-511) | Replace the conditional operator (?:) with mid() | Major |
+| [TUBEMQ-512](https://issues.apache.org/jira/browse/TUBEMQ-512) | Add package length control based on Topic | Major |
+| [TUBEMQ-515](https://issues.apache.org/jira/browse/TUBEMQ-515) | Add cluster Topic view web api | Major |
+
+### BUG FIXES:
+| JIRA | Summary | Priority |
+| :---- | :------- | :------- |
+| [TUBEMQ-441](https://issues.apache.org/jira/browse/TUBEMQ-441) | An error occurred when using the Tubemq class to create a sink table | Major |
+| [TUBEMQ-437](https://issues.apache.org/jira/browse/TUBEMQ-437) | Fix tubemq table source sink factory instance creating problem | Major |
+| [TUBEMQ-443](https://issues.apache.org/jira/browse/TUBEMQ-443) | TubemqSourceFunction class prints too many logs problem | Major |
+| [TUBEMQ-446](https://issues.apache.org/jira/browse/TUBEMQ-446) | Small bugs fix that do not affect the main logics | Major |
+| [TUBEMQ-453](https://issues.apache.org/jira/browse/TUBEMQ-453) | TubemqSourceFunction class prints too many logs problem | Major |
+| [TUBEMQ-457](https://issues.apache.org/jira/browse/TUBEMQ-457) | There is no need to return StringBuilder in Master.java | Major |
+| [TUBEMQ-495](https://issues.apache.org/jira/browse/TUBEMQ-495) | Code implementation adjustment based on SpotBugs check | Major |
+| [TUBEMQ-506](https://issues.apache.org/jira/browse/TUBEMQ-506) | cmakelist error | Major |
+| [TUBEMQ-510](https://issues.apache.org/jira/browse/TUBEMQ-510) | Found a bug in MessageProducerExample class | Major |
+| [TUBEMQ-518](https://issues.apache.org/jira/browse/TUBEMQ-518) | fix parameter pass error | Major |
+
+### SUB-TASK:
+
+| JIRA | Summary | Priority |
+| :---- | :------- | :------- |
+| [TUBEMQ-433](https://issues.apache.org/jira/browse/TUBEMQ-433) | add tubemq perf-consumer/producer scripts | Major |
+| [TUBEMQ-434](https://issues.apache.org/jira/browse/TUBEMQ-434) | Adjust Broker API mapping | Major |
+| [TUBEMQ-435](https://issues.apache.org/jira/browse/TUBEMQ-435) | Create Web field Mapping | Major |
+| [TUBEMQ-436](https://issues.apache.org/jira/browse/TUBEMQ-436) | Adjust Broker's HTTP API implementation | Major |
+| [TUBEMQ-439](https://issues.apache.org/jira/browse/TUBEMQ-439) | Add Cli field Scheme definition | Major |
+| [TUBEMQ-444](https://issues.apache.org/jira/browse/TUBEMQ-444) | Add consume and produce Cli commands | Major |
+| [TUBEMQ-447](https://issues.apache.org/jira/browse/TUBEMQ-447) | Add Broker-Admin Cli | Major |
+| [TUBEMQ-452](https://issues.apache.org/jira/browse/TUBEMQ-452) | Optimize rebalance performance | Major |
+| [TUBEMQ-464](https://issues.apache.org/jira/browse/TUBEMQ-464) | Add parameter rebalanceParallel in master.ini | Major |
+| [TUBEMQ-467](https://issues.apache.org/jira/browse/TUBEMQ-467) | Add WEB APIs of Master and Broker | Major |
+| [TUBEMQ-471](https://issues.apache.org/jira/browse/TUBEMQ-471) | Add query API Introduction of TopicName and BrokerId collection | Major |
+| [TUBEMQ-494](https://issues.apache.org/jira/browse/TUBEMQ-494) | Update API interface instruction document | Major |
+
## Release 0.7.0-incubating - Released (as of 2020-11-25)
### New Features:
[incubator-tubemq] 22/49: [TUBEMQ-482] Add offset query api
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit d8580f21b44e4866b5f63352afd7664b7e67bfe4
Author: gosonzhang <go...@tencent.com>
AuthorDate: Fri Dec 25 16:37:14 2020 +0800
[TUBEMQ-482] Add offset query api
---
.../server/broker/metadata/TopicMetadata.java | 29 ++++++++
.../server/broker/msgstore/MessageStore.java | 11 ++-
.../broker/msgstore/MessageStoreManager.java | 62 +++++++++++++++-
.../server/broker/msgstore/StoreService.java | 6 ++
.../server/broker/offset/DefaultOffsetManager.java | 46 +++++++-----
.../tubemq/server/broker/offset/OffsetService.java | 6 +-
.../server/broker/utils/GroupOffsetInfo.java | 85 ++++++++++++++++++++++
.../server/broker/utils/TopicPubStoreInfo.java | 45 ++++++++++++
.../server/broker/web/BrokerAdminServlet.java | 79 +++++++++++++++-----
9 files changed, 323 insertions(+), 46 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
index c582606..800254b 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
@@ -17,7 +17,9 @@
package org.apache.tubemq.server.broker.metadata;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import org.apache.tubemq.corebase.TBaseConstants;
@@ -251,6 +253,33 @@ public class TopicMetadata {
return partIds;
}
+ // builder the partitionId set for each store
+ public Map<Integer, Set<Integer>> getStorePartIdMap() {
+ Map<Integer, Set<Integer>> storePartIds = new HashMap<>();
+ for (int i = 0; i < numTopicStores; i++) {
+ Set<Integer> partIds = new HashSet<>();
+ for (int j = 0; j < numPartitions; j++) {
+ partIds.add(i * TBaseConstants.META_STORE_INS_BASE + j);
+ }
+ storePartIds.put(i, partIds);
+ }
+ return storePartIds;
+ }
+
+ public int getStoreIdByPartitionId(int partitionId) {
+ return partitionId % TBaseConstants.META_STORE_INS_BASE;
+ }
+
+ public Set<Integer> getPartIdsByStoreId(int storeId) {
+ Set<Integer> partIds = new HashSet<>();
+ if (storeId >= 0 && storeId < numTopicStores) {
+ for (int i = 0; i < numPartitions; i++) {
+ partIds.add(storeId * TBaseConstants.META_STORE_INS_BASE + i);
+ }
+ }
+ return partIds;
+ }
+
public int getStatusId() {
return statusId;
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
index a827723..e610a8b 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStore.java
@@ -133,9 +133,9 @@ public class MessageStore implements Closeable {
this.writeCacheFlushIntvl = topicMetadata.getMemCacheFlushIntvl();
int tmpIndexReadCnt = tubeConfig.getIndexTransCount() * partitionNum;
memMaxIndexReadCnt.set(tmpIndexReadCnt <= 6000
- ? 6000 : (tmpIndexReadCnt >= 10000 ? 10000 : tmpIndexReadCnt));
+ ? 6000 : (Math.min(tmpIndexReadCnt, 10000)));
fileMaxIndexReadCnt.set(tmpIndexReadCnt < 8000
- ? 8000 : (tmpIndexReadCnt >= 13500 ? 13500 : tmpIndexReadCnt));
+ ? 8000 : (Math.min(tmpIndexReadCnt, 13500)));
memMaxFilterIndexReadCnt.set(memMaxIndexReadCnt.get() * 2);
fileMaxFilterIndexReadCnt.set(fileMaxIndexReadCnt.get() * 3);
fileLowReqMaxFilterIndexReadCnt.set(fileMaxFilterIndexReadCnt.get() * 10);
@@ -250,8 +250,7 @@ public class MessageStore implements Closeable {
}
}
// before read from file, adjust request's offset.
- long reqNewOffset = requestOffset < this.msgFileStore.getIndexMinOffset()
- ? this.msgFileStore.getIndexMinOffset() : requestOffset;
+ long reqNewOffset = Math.max(requestOffset, this.msgFileStore.getIndexMinOffset());
if (reqSwitch <= 1 && reqNewOffset >= getFileIndexMaxOffset()) {
return new GetMessageResult(false, TErrCodeConstants.NOT_FOUND,
reqNewOffset, 0, "current offset is exceed max file offset");
@@ -409,9 +408,9 @@ public class MessageStore implements Closeable {
maxFileValidDurMs.set(parseDeletePolicy(topicMetadata.getDeletePolicy()));
int tmpIndexReadCnt = tubeConfig.getIndexTransCount() * partitionNum;
memMaxIndexReadCnt.set(tmpIndexReadCnt <= 6000
- ? 6000 : (tmpIndexReadCnt >= 10000 ? 10000 : tmpIndexReadCnt));
+ ? 6000 : (Math.min(tmpIndexReadCnt, 10000)));
fileMaxIndexReadCnt.set(tmpIndexReadCnt < 8000
- ? 8000 : (tmpIndexReadCnt >= 13500 ? 13500 : tmpIndexReadCnt));
+ ? 8000 : (Math.min(tmpIndexReadCnt, 13500)));
memMaxFilterIndexReadCnt.set(memMaxIndexReadCnt.get() * 2);
fileMaxFilterIndexReadCnt.set(fileMaxIndexReadCnt.get() * 3);
fileLowReqMaxFilterIndexReadCnt.set(fileMaxFilterIndexReadCnt.get() * 10);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStoreManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStoreManager.java
index 6275f3a..2d55ba1 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStoreManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/MessageStoreManager.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -51,6 +52,7 @@ import org.apache.tubemq.server.broker.metadata.TopicMetadata;
import org.apache.tubemq.server.broker.msgstore.disk.GetMessageResult;
import org.apache.tubemq.server.broker.nodeinfo.ConsumerNodeInfo;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
+import org.apache.tubemq.server.broker.utils.TopicPubStoreInfo;
import org.apache.tubemq.server.common.TStatusConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,8 +92,7 @@ public class MessageStoreManager implements StoreService {
this.metadataManager = this.tubeBroker.getMetadataManager();
this.isRemovingTopic.set(false);
this.maxMsgTransferSize =
- tubeConfig.getTransferSize() > DataStoreUtils.MAX_MSG_TRANSFER_SIZE
- ? DataStoreUtils.MAX_MSG_TRANSFER_SIZE : tubeConfig.getTransferSize();
+ Math.min(tubeConfig.getTransferSize(), DataStoreUtils.MAX_MSG_TRANSFER_SIZE);
this.metadataManager.addPropertyChangeListener("topicConfigMap", new PropertyChangeListener() {
@Override
public void propertyChange(final PropertyChangeEvent evt) {
@@ -385,6 +386,63 @@ public class MessageStoreManager implements StoreService {
return Collections.unmodifiableMap(this.dataStores);
}
+ /***
+ * Query topic's publish info.
+ *
+ * @param topicSet query's topic set
+ *
+ * @return the topic's offset info
+ */
+ @Override
+ public Map<String, Map<Integer, TopicPubStoreInfo>> getTopicPublishInfos(
+ Set<String> topicSet) {
+ MessageStore store = null;
+ TopicMetadata topicMetadata = null;
+ Set<String> qryTopicSet = new HashSet<>();
+ Map<String, Map<Integer, TopicPubStoreInfo>> topicPubStoreInfoMap = new HashMap<>();
+ Map<String, TopicMetadata> confTopicInfo = metadataManager.getTopicConfigMap();
+ if (topicSet == null || topicSet.isEmpty()) {
+ qryTopicSet.addAll(confTopicInfo.keySet());
+ } else {
+ for (String topic : topicSet) {
+ if (confTopicInfo.containsKey(topic)) {
+ qryTopicSet.add(topic);
+ }
+ }
+ }
+ if (qryTopicSet.isEmpty()) {
+ return topicPubStoreInfoMap;
+ }
+ for (String topic : qryTopicSet) {
+ topicMetadata = confTopicInfo.get(topic);
+ if (topicMetadata == null) {
+ continue;
+ }
+ Map<Integer, MessageStore> storeMap = dataStores.get(topic);
+ if (storeMap == null) {
+ continue;
+ }
+ Map<Integer, TopicPubStoreInfo> storeInfoMap = new HashMap<>();
+ for (Map.Entry<Integer, MessageStore> entry : storeMap.entrySet()) {
+ if (entry == null
+ || entry.getKey() == null
+ || entry.getValue() == null) {
+ continue;
+ }
+ store = entry.getValue();
+ for (Integer partitionId : topicMetadata.getPartIdsByStoreId(entry.getKey())) {
+ TopicPubStoreInfo storeInfo =
+ new TopicPubStoreInfo(topic, entry.getKey(), partitionId,
+ store.getIndexMinOffset(), store.getIndexMaxOffset(),
+ store.getDataMinOffset(), store.getDataMaxOffset());
+ storeInfoMap.put(partitionId, storeInfo);
+ }
+ }
+ topicPubStoreInfoMap.put(topic, storeInfoMap);
+ }
+ return topicPubStoreInfoMap;
+ }
+
private Set<File> getLogDirSet(final BrokerConfig tubeConfig) throws IOException {
TopicMetadata topicMetadata = null;
final Set<String> paths = new HashSet<>();
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/StoreService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/StoreService.java
index d5f7f32..184fb4d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/StoreService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/StoreService.java
@@ -20,6 +20,10 @@ package org.apache.tubemq.server.broker.msgstore;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.tubemq.server.broker.utils.TopicPubStoreInfo;
+
/***
* Store service interface.
@@ -35,4 +39,6 @@ public interface StoreService {
MessageStore getOrCreateMessageStore(final String topic,
final int partition) throws Throwable;
+
+ Map<String, Map<Integer, TopicPubStoreInfo>> getTopicPublishInfos(Set<String> topicSet);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
index bdd85b3..df3afc4 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.daemon.AbstractDaemonService;
import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.broker.BrokerConfig;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
@@ -399,22 +400,26 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
* @return group offset info in memory or zk
*/
@Override
- public Map<String, Map<Integer, Long>> queryGroupOffset(
+ public Map<String, Map<Integer, Tuple2<Long, Long>>> queryGroupOffset(
String group, Map<String, Set<Integer>> topicPartMap) {
- Map<String, Map<Integer, Long>> result = new HashMap<>();
+ Map<String, Map<Integer, Tuple2<Long, Long>>> result = new HashMap<>();
// search group from memory
Map<String, OffsetStorageInfo> topicPartOffsetMap = cfmOffsetMap.get(group);
if (topicPartOffsetMap == null) {
// query from zookeeper
for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ if (entry == null || entry.getKey() == null || entry.getValue() == null) {
+ continue;
+ }
Map<Integer, Long> qryResult =
- zkOffsetStorage.queryGroupOffsetInfo(
- group, entry.getKey(), entry.getValue());
- Map<Integer, Long> offsetMap = new HashMap<>();
+ zkOffsetStorage.queryGroupOffsetInfo(group,
+ entry.getKey(), entry.getValue());
+ Map<Integer, Tuple2<Long, Long>> offsetMap = new HashMap<>();
for (Map.Entry<Integer, Long> item : qryResult.entrySet()) {
- if (item.getValue() != null) {
- offsetMap.put(item.getKey(), item.getValue());
+ if (item == null || item.getKey() == null || item.getValue() == null) {
+ continue;
}
+ offsetMap.put(item.getKey(), new Tuple2<>(item.getValue(), 0L));
}
if (!offsetMap.isEmpty()) {
result.put(entry.getKey(), offsetMap);
@@ -422,14 +427,18 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
}
} else {
// found in memory, get offset values
+ Map<String, Long> tmpPartOffsetMap = tmpOffsetMap.get(group);
for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
- Map<Integer, Long> offsetMap = new HashMap<>();
+ Map<Integer, Tuple2<Long, Long>> offsetMap = new HashMap<>();
for (Integer partitionId : entry.getValue()) {
String offsetCacheKey =
getOffsetCacheKey(entry.getKey(), partitionId);
OffsetStorageInfo offsetInfo = topicPartOffsetMap.get(offsetCacheKey);
+ Long tmpOffset = tmpPartOffsetMap.get(offsetCacheKey);
if (offsetInfo != null) {
- offsetMap.put(partitionId, offsetInfo.getOffset());
+ offsetMap.put(partitionId,
+ new Tuple2<>(offsetInfo.getOffset(),
+ (tmpOffset == null ? 0 : tmpOffset)));
}
}
if (!offsetMap.isEmpty()) {
@@ -451,9 +460,9 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
* @return at least one record modified
*/
@Override
- public boolean modifyGroupOffset(MessageStoreManager storeManager, Set<String> groups,
- Map<String, Map<Integer, Long>> topicPartOffsetMap,
- String modifier) {
+ public boolean modifyGroupOffset(
+ MessageStoreManager storeManager, Set<String> groups,
+ Map<String, Map<Integer, Tuple2<Long, Long>>> topicPartOffsetMap, String modifier) {
long oldOffset = -1;
long reSetOffset = -1;
boolean changed = false;
@@ -461,17 +470,18 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
StringBuilder strBuidler = new StringBuilder(512);
// set offset by group
for (String group : groups) {
- for (Map.Entry<String, Map<Integer, Long>> entry : topicPartOffsetMap.entrySet()) {
- Map<Integer, Long> partOffsetMap = entry.getValue();
+ for (Map.Entry<String, Map<Integer, Tuple2<Long, Long>>> entry
+ : topicPartOffsetMap.entrySet()) {
+ Map<Integer, Tuple2<Long, Long>> partOffsetMap = entry.getValue();
if (partOffsetMap == null) {
continue;
}
// set offset
- for (Map.Entry<Integer, Long> entry1 : partOffsetMap.entrySet()) {
+ for (Map.Entry<Integer, Tuple2<Long, Long>> entry1 : partOffsetMap.entrySet()) {
if (entry1.getValue() == null) {
continue;
}
- reSetOffset = entry1.getValue();
+ Tuple2<Long, Long> offsetTuple = entry1.getValue();
// get topic store
try {
store = storeManager.getOrCreateMessageStore(
@@ -485,8 +495,8 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
long firstOffset = store.getIndexMinOffset();
long lastOffset = store.getIndexMaxOffset();
// adjust reseted offset value
- reSetOffset = reSetOffset < firstOffset
- ? firstOffset : Math.min(reSetOffset, lastOffset);
+ reSetOffset = offsetTuple.f0 < firstOffset
+ ? firstOffset : Math.min(offsetTuple.f0, lastOffset);
String offsetCacheKey =
getOffsetCacheKey(entry.getKey(), entry1.getKey());
getAndResetTmpOffset(group, offsetCacheKey);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
index 05f0724..fcebdfc 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
@@ -19,6 +19,8 @@ package org.apache.tubemq.server.broker.offset;
import java.util.Map;
import java.util.Set;
+
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorageInfo;
@@ -63,10 +65,10 @@ public interface OffsetService {
Set<String> getGroupSubInfo(String group);
- Map<String, Map<Integer, Long>> queryGroupOffset(
+ Map<String, Map<Integer, Tuple2<Long, Long>>> queryGroupOffset(
String group, Map<String, Set<Integer>> topicPartMap);
boolean modifyGroupOffset(MessageStoreManager storeManager, Set<String> groups,
- Map<String, Map<Integer, Long>> topicPartOffsetMap,
+ Map<String, Map<Integer, Tuple2<Long, Long>>> topicPartOffsetMap,
String modifier);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
new file mode 100644
index 0000000..9a4abe3
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/GroupOffsetInfo.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.broker.utils;
+
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.utils.Tuple2;
+
+
+public class GroupOffsetInfo {
+ public int partitionId = TBaseConstants.META_VALUE_UNDEFINED;
+ public long offsetMin = TBaseConstants.META_VALUE_UNDEFINED;
+ public long offsetMax = TBaseConstants.META_VALUE_UNDEFINED;
+ public long dataMin = TBaseConstants.META_VALUE_UNDEFINED;
+ public long dataMax = TBaseConstants.META_VALUE_UNDEFINED;
+ public long curOffset = TBaseConstants.META_VALUE_UNDEFINED;
+ public long flightOffset = TBaseConstants.META_VALUE_UNDEFINED;
+ public long offsetLag = TBaseConstants.META_VALUE_UNDEFINED;
+ public long curDataOffset = TBaseConstants.META_VALUE_UNDEFINED;
+ public long dataLag = TBaseConstants.META_VALUE_UNDEFINED;
+
+ public GroupOffsetInfo(int partitionId) {
+ this.partitionId = partitionId;
+ }
+
+ public void setPartPubStoreInfo(TopicPubStoreInfo pubStoreInfo) {
+ if (pubStoreInfo != null) {
+ this.offsetMin = pubStoreInfo.indexStart;
+ this.offsetMax = pubStoreInfo.indexEnd;
+ this.dataMin = pubStoreInfo.dataStart;
+ this.dataMax = pubStoreInfo.dataEnd;
+ }
+ }
+
+ public void setConsumeOffsetInfo(Tuple2<Long, Long> offsetInfo) {
+ if (offsetInfo != null) {
+ this.curOffset = offsetInfo.f0;
+ this.flightOffset = offsetInfo.f1;
+ }
+ }
+
+ public void setConsumeDataOffsetInfo(long curDataOffset) {
+ if (curDataOffset >= 0) {
+ this.curDataOffset = curDataOffset;
+ }
+ }
+
+ public void calculateLag() {
+ if (offsetMax != TBaseConstants.META_VALUE_UNDEFINED
+ && curOffset != TBaseConstants.META_VALUE_UNDEFINED) {
+ offsetLag = offsetMax - curOffset;
+ }
+ if (dataMax != TBaseConstants.META_VALUE_UNDEFINED
+ && curDataOffset != TBaseConstants.META_VALUE_UNDEFINED) {
+ dataLag = dataMax - curDataOffset;
+ }
+ }
+
+ public StringBuilder buildOffsetInfo(StringBuilder sBuilder) {
+ sBuilder.append("{\"partitionId\":").append(partitionId)
+ .append(",\"curOffset\":").append(curOffset)
+ .append(",\"flightOffset\":").append(flightOffset)
+ .append(",\"curDataOffset\":").append(curDataOffset)
+ .append(",\"offsetLag\":").append(offsetLag)
+ .append(",\"dataLag\":").append(dataLag)
+ .append(",\"offsetMax\":").append(offsetMax)
+ .append(",\"dataMax\":").append(dataMax)
+ .append("}");
+ return sBuilder;
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java
new file mode 100644
index 0000000..b2257dd
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/TopicPubStoreInfo.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.broker.utils;
+
+import org.apache.tubemq.corebase.TBaseConstants;
+
+
+
+public class TopicPubStoreInfo {
+
+ public String topicName = null;
+ public int storeId = TBaseConstants.META_VALUE_UNDEFINED;
+ public int partitionId = TBaseConstants.META_VALUE_UNDEFINED;
+ public long indexStart = 0L;
+ public long indexEnd = 0L;
+ public long dataStart = 0L;
+ public long dataEnd = 0L;
+
+ public TopicPubStoreInfo(String topicName, int storeId, int partitionId,
+ long indexStart, long indexEnd, long dataStart, long dataEnd) {
+ this.topicName = topicName;
+ this.storeId = storeId;
+ this.partitionId = partitionId;
+ this.indexStart = indexStart;
+ this.indexEnd = indexEnd;
+ this.dataStart = dataStart;
+ this.dataEnd = dataEnd;
+ }
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index 91bfd23..9a0506e 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -27,12 +27,15 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.Tuple2;
import org.apache.tubemq.server.broker.TubeBroker;
import org.apache.tubemq.server.broker.metadata.TopicMetadata;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.nodeinfo.ConsumerNodeInfo;
import org.apache.tubemq.server.broker.offset.OffsetService;
+import org.apache.tubemq.server.broker.utils.GroupOffsetInfo;
+import org.apache.tubemq.server.broker.utils.TopicPubStoreInfo;
import org.apache.tubemq.server.common.fielddef.WebFieldDef;
import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.common.utils.WebParameterUtils;
@@ -672,42 +675,34 @@ public class BrokerAdminServlet extends AbstractWebHandler {
Set<String> topicSet = (Set<String>) result.retData1;
// verify the acquired Topic set and
// query the corresponding offset information
- Map<String, Map<String, Map<Integer, Long>>> groupOffsetMaps = new HashMap<>();
- for (String group : qryGroupNameSet) {
- Map<String, Set<Integer>> topicPartMap =
- validAndGetPartitions(group, topicSet);
- Map<String, Map<Integer, Long>> groupOffsetMap =
- broker.getOffsetManager().queryGroupOffset(group, topicPartMap);
- groupOffsetMaps.put(group, groupOffsetMap);
- }
+ Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> groupOffsetMaps =
+ getGroupOffsetInfo(qryGroupNameSet, topicSet);
// builder result
int totalCnt = 0;
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
- for (Map.Entry<String, Map<String, Map<Integer, Long>>> entry
+ for (Map.Entry<String, Map<String, Map<Integer, GroupOffsetInfo>>> entry
: groupOffsetMaps.entrySet()) {
if (totalCnt++ > 0) {
sBuilder.append(",");
}
- Map<String, Map<Integer, Long>> topicPartMap = entry.getValue();
+ Map<String, Map<Integer, GroupOffsetInfo>> topicPartMap = entry.getValue();
sBuilder.append("{\"groupName\":\"").append(entry.getKey())
.append("\",\"subInfo\":[");
int topicCnt = 0;
- for (Map.Entry<String, Map<Integer, Long>> entry1 : topicPartMap.entrySet()) {
+ for (Map.Entry<String, Map<Integer, GroupOffsetInfo>> entry1 : topicPartMap.entrySet()) {
if (topicCnt++ > 0) {
sBuilder.append(",");
}
- Map<Integer, Long> partOffMap = entry1.getValue();
+ Map<Integer, GroupOffsetInfo> partOffMap = entry1.getValue();
sBuilder.append("{\"topicName\":\"").append(entry1.getKey())
.append("\",\"offsets\":[");
int partCnt = 0;
- for (Map.Entry<Integer, Long> entry2 : partOffMap.entrySet()) {
+ for (Map.Entry<Integer, GroupOffsetInfo> entry2 : partOffMap.entrySet()) {
if (partCnt++ > 0) {
sBuilder.append(",");
}
- sBuilder.append("{\"").append(this.broker.getTubeConfig().getBrokerId())
- .append(TokenConstants.ATTR_SEP).append(entry1.getKey())
- .append(TokenConstants.ATTR_SEP).append(entry2.getKey())
- .append("\":").append(entry2.getValue()).append("}");
+ GroupOffsetInfo offsetInfo = entry2.getValue();
+ offsetInfo.buildOffsetInfo(sBuilder);
}
sBuilder.append("],\"partCount\":").append(partCnt).append("}");
}
@@ -777,7 +772,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
return;
}
// query offset from source group
- Map<String, Map<Integer, Long>> srcGroupOffsets =
+ Map<String, Map<Integer, Tuple2<Long, Long>>> srcGroupOffsets =
broker.getOffsetManager().queryGroupOffset(srcGroupName, topicPartMap);
boolean changed = broker.getOffsetManager().modifyGroupOffset(
broker.getStoreManager(), tgtGroupNameSet, srcGroupOffsets, modifier);
@@ -785,6 +780,48 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
}
+ // builder group's offset info
+ private Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> getGroupOffsetInfo(
+ Set<String> groupSet, Set<String> topicSet) {
+ long curReadDataOffset = -2;
+ long curDataLag = -2;
+ Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> groupOffsetMaps = new HashMap<>();
+ for (String group : groupSet) {
+ Map<String, Map<Integer, GroupOffsetInfo>> topicOffsetRet = new HashMap<>();
+ // valid and get topic's partitionIds
+ Map<String, Set<Integer>> topicPartMap = validAndGetPartitions(group, topicSet);
+ // get topic's publish info
+ Map<String, Map<Integer, TopicPubStoreInfo>> topicStorePubInfoMap =
+ broker.getStoreManager().getTopicPublishInfos(topicPartMap.keySet());
+ // get group's booked offset info
+ Map<String, Map<Integer, Tuple2<Long, Long>>> groupOffsetMap =
+ broker.getOffsetManager().queryGroupOffset(group, topicPartMap);
+ // get offset info array
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ String topic = entry.getKey();
+ Map<Integer, GroupOffsetInfo> partOffsetRet = new HashMap<>();
+ Map<Integer, TopicPubStoreInfo> storeInfoMap = topicStorePubInfoMap.get(topic);
+ Map<Integer, Tuple2<Long, Long>> partBookedMap = groupOffsetMap.get(topic);
+ for (Integer partitionId : entry.getValue()) {
+ GroupOffsetInfo offsetInfo = new GroupOffsetInfo(partitionId);
+ offsetInfo.setPartPubStoreInfo(storeInfoMap.get(partitionId));
+ offsetInfo.setConsumeOffsetInfo(partBookedMap.get(partitionId));
+ String queryKey = buildQueryID(group, topic, partitionId);
+ ConsumerNodeInfo nodeInfo = broker.getConsumerNodeInfo(queryKey);
+ if (nodeInfo != null) {
+ offsetInfo.setConsumeDataOffsetInfo(nodeInfo.getLastDataRdOffset());
+ }
+ offsetInfo.calculateLag();
+ partOffsetRet.put(partitionId, offsetInfo);
+ }
+ topicOffsetRet.put(topic, partOffsetRet);
+ }
+ groupOffsetMaps.put(group, topicOffsetRet);
+ }
+ return groupOffsetMaps;
+ }
+
+
private Map<String, Set<Integer>> validAndGetPartitions(String group, Set<String> topicSet) {
Map<String, Set<Integer>> topicPartMap = new HashMap<>();
// query stored topic set stored in memory or zk
@@ -807,4 +844,10 @@ public class BrokerAdminServlet extends AbstractWebHandler {
return topicPartMap;
}
+ private String buildQueryID(String group, String topic, int partitionId) {
+ return new StringBuilder(512).append(group)
+ .append(TokenConstants.ATTR_SEP).append(topic)
+ .append(TokenConstants.ATTR_SEP).append(partitionId).toString();
+ }
+
}
[incubator-tubemq] 26/49: [TUBEMQ-486]Add the delete API of
consumer group offset
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 7dd7268568e51b87af15c0021506ccfb60ee6da6
Author: gosonzhang <go...@tencent.com>
AuthorDate: Wed Jan 6 11:33:28 2021 +0800
[TUBEMQ-486]Add the delete API of consumer group offset
---
.../java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index 1f2fb5d..30d7247 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -1104,7 +1104,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
for (Integer partitionId : entry.getValue()) {
GroupOffsetInfo offsetInfo = new GroupOffsetInfo(partitionId);
offsetInfo.setPartPubStoreInfo(
- storeInfoMap == null ? null :storeInfoMap.get(partitionId));
+ storeInfoMap == null ? null : storeInfoMap.get(partitionId));
offsetInfo.setConsumeOffsetInfo(
partBookedMap == null ? null : partBookedMap.get(partitionId));
String queryKey = buildQueryID(group, topic, partitionId);
[incubator-tubemq] 47/49: [TUBEMQ-546]Restore the original license
header of the referenced external source files
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 9530caec89fc123c5e2e91f60def89b7c64789fe
Author: gosonzhang <go...@tencent.com>
AuthorDate: Thu Jan 28 16:55:44 2021 +0800
[TUBEMQ-546]Restore the original license header of the referenced external source files
---
pom.xml | 5 +++-
tubemq-client-twins/tubemq-client-cpp/src/any.h | 21 +----------------
tubemq-client-twins/tubemq-client-cpp/src/buffer.h | 22 +++---------------
.../apache/tubemq/corebase/utils/TStringUtils.java | 27 +++++++++++++---------
4 files changed, 24 insertions(+), 51 deletions(-)
diff --git a/pom.xml b/pom.xml
index bd51de2..a1e5478 100644
--- a/pom.xml
+++ b/pom.xml
@@ -265,7 +265,10 @@
<exclude>resources/assets/public/**</exclude>
<exclude>DISCLAIMER</exclude>
<exclude>tubemq-web/**</exclude>
-
+ <!-- copy or modify files from other projects -->
+ <exclude>**/tubemq-client-twins/tubemq-client-cpp/src/any.h</exclude>
+ <exclude>**/tubemq-client-twins/tubemq-client-cpp/src/buffer.h</exclude>
+ <exclude>**/tubemq-core/src/corebase/utils/TStringUtils.java</exclude>
<exclude>**/tubemq-client-twins/tubemq-client-cpp/third_party/**</exclude>
</excludes>
</configuration>
diff --git a/tubemq-client-twins/tubemq-client-cpp/src/any.h b/tubemq-client-twins/tubemq-client-cpp/src/any.h
index 4bb2a15..dd6eaa5 100644
--- a/tubemq-client-twins/tubemq-client-cpp/src/any.h
+++ b/tubemq-client-twins/tubemq-client-cpp/src/any.h
@@ -1,24 +1,5 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
// Copy from evpp project
-// @see https://github.com/Qihoo360/evpp/blob/master/evpp/any.h
+// @see https://github.com/Qihoo360/evpp/blob/master/evpp/any.h commit version c5038a6
#ifndef TUBEMQ_CLIENT_CPP_SRC_ANY_H_
#define TUBEMQ_CLIENT_CPP_SRC_ANY_H_
diff --git a/tubemq-client-twins/tubemq-client-cpp/src/buffer.h b/tubemq-client-twins/tubemq-client-cpp/src/buffer.h
index da8c95b..fc8425d 100644
--- a/tubemq-client-twins/tubemq-client-cpp/src/buffer.h
+++ b/tubemq-client-twins/tubemq-client-cpp/src/buffer.h
@@ -1,24 +1,8 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
+// Modified from muduo project http://github.com/chenshuo/muduo
+// @see https://github.com/chenshuo/muduo/blob/master/muduo/net/Buffer.h and https://github.com/chenshuo/muduo/blob/master/muduo/net/Buffer.cc
// Modified from evpp
-// @see https://github.com/Qihoo360/evpp/blob/master/evpp/buffer.h
+// @see https://github.com/Qihoo360/evpp/blob/master/evpp/buffer.h, commit version b2535d7
#ifndef _TUBEMQ_BUFFER_H_
#define _TUBEMQ_BUFFER_H_
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/TStringUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/TStringUtils.java
index 985dd5d..f99ba52 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/TStringUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/TStringUtils.java
@@ -1,13 +1,13 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+/*
+ * Copyright (c) 2002-2012 Alibaba Group Holding Limited.
+ * All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,6 +15,12 @@
* limitations under the License.
*/
+/*
+ * Modified from citrus <a href="https://github.com/webx/citrus">citrus Project</a>
+ * file address: https://github.com/webx/citrus/blob/r3.1.4/common/util/src/
+ * main/java/com/alibaba/citrus/util/StringUtil.java
+ */
+
package org.apache.tubemq.corebase.utils;
import java.io.PrintWriter;
@@ -28,7 +34,6 @@ import org.apache.tubemq.corebase.TokenConstants;
/**
* Utility to String operations.
- * Modified version of <a href="https://github.com/webx/citrus">citrus Project</a>
*/
public class TStringUtils {
[incubator-tubemq] 08/49: [TUBEMQ-443] TubemqSourceFunction class
prints too many logs problem (#344)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit aec25b501feb8f5a62832d0403e9589593f14bf3
Author: leno1001 <19...@qq.com>
AuthorDate: Wed Dec 9 11:15:05 2020 +0800
[TUBEMQ-443] TubemqSourceFunction class prints too many logs problem (#344)
Co-authored-by: 曹显乐 <xi...@vivo.com>
---
.../flink/connectors/tubemq/TubemqSourceFunction.java | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java
index 2f375a6..c93e057 100644
--- a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java
+++ b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/TubemqSourceFunction.java
@@ -254,9 +254,12 @@ public class TubemqSourceFunction<T>
ConsumerResult consumeResult = messagePullConsumer.getMessage();
if (!consumeResult.isSuccess()) {
- LOG.info("Could not consume messages from tubemq (errcode: {}, " +
- "errmsg: {}).", consumeResult.getErrCode(),
- consumeResult.getErrMsg());
+ if (!(consumeResult.getErrCode() == 400 || consumeResult.getErrCode() == 404 ||
+ consumeResult.getErrCode() == 405 || consumeResult.getErrCode() == 406 ||
+ consumeResult.getErrCode() == 407 || consumeResult.getErrCode() == 408)) {
+ LOG.info("Could not consume messages from tubemq (errcode: {}, " + "errmsg: {}).",
+ consumeResult.getErrCode(), consumeResult.getErrMsg());
+ }
Duration idleTime =
Duration.between(lastConsumeInstant, Instant.now());
@@ -297,9 +300,12 @@ public class TubemqSourceFunction<T>
messagePullConsumer
.confirmConsume(consumeResult.getConfirmContext(), true);
if (!confirmResult.isSuccess()) {
- LOG.warn("Could not confirm messages to tubemq (errcode: {}, " +
- "errmsg: {}).", confirmResult.getErrCode(),
- confirmResult.getErrMsg());
+ if (!(confirmResult.getErrCode() == 400 || confirmResult.getErrCode() == 404 ||
+ confirmResult.getErrCode() == 405 || confirmResult.getErrCode() == 406 ||
+ confirmResult.getErrCode() == 407 || confirmResult.getErrCode() == 408)) {
+ LOG.warn("Could not confirm messages to tubemq (errcode: {}, " + "errmsg: {}).",
+ confirmResult.getErrCode(), confirmResult.getErrMsg());
+ }
}
}
}
[incubator-tubemq] 34/49: [TUBEMQ-509] Adjust the packet length
check when data is loaded
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 1b7ca30a51e65e3d2c4d3b0ef8e64788788d6dca
Author: gosonzhang <go...@tencent.com>
AuthorDate: Tue Jan 12 14:30:06 2021 +0800
[TUBEMQ-509] Adjust the packet length check when data is loaded
---
.../apache/tubemq/server/broker/metadata/ClusterConfigHolder.java | 8 --------
.../apache/tubemq/server/broker/msgstore/disk/FileSegment.java | 5 ++---
.../apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java | 3 +--
.../org/apache/tubemq/server/broker/utils/DataStoreUtils.java | 7 +++----
4 files changed, 6 insertions(+), 17 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
index c72427d..134e2ef 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/ClusterConfigHolder.java
@@ -33,10 +33,6 @@ public class ClusterConfigHolder {
+ TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE);
private static AtomicInteger minMemCacheSize =
new AtomicInteger(TBaseConstants.META_MIN_MEM_BUFFER_SIZE);
- private static AtomicInteger maxMsgStoreLength =
- new AtomicInteger(TBaseConstants.META_MAX_MESSAGE_DATA_SIZE
- + TBaseConstants.META_MAX_MESSAGE_HEADER_SIZE
- + DataStoreUtils.STORE_DATA_HEADER_LEN);
public ClusterConfigHolder() {
@@ -58,7 +54,6 @@ public class ClusterConfigHolder {
maxMsgSize.set(tmpMaxSize);
minMemCacheSize.set(tmpMaxSize +
(tmpMaxSize % 4 + 1) * TBaseConstants.META_MESSAGE_SIZE_ADJUST);
- maxMsgStoreLength.set(tmpMaxSize + DataStoreUtils.STORE_DATA_HEADER_LEN);
}
}
}
@@ -76,7 +71,4 @@ public class ClusterConfigHolder {
return minMemCacheSize.get();
}
- public static int getMaxMsgStoreLength() {
- return maxMsgStoreLength.get();
- }
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java
index 6fda352..949d149 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/FileSegment.java
@@ -25,7 +25,6 @@ import java.nio.channels.FileChannel;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.tubemq.corebase.utils.CheckSum;
-import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -366,7 +365,7 @@ public class FileSegment implements Segment {
itemNext = validBytes + DataStoreUtils.STORE_DATA_HEADER_LEN + itemMsglen;
if ((itemMsgToken != DataStoreUtils.STORE_DATA_TOKER_BEGIN_VALUE)
|| (itemMsglen <= 0)
- || (itemMsglen > ClusterConfigHolder.getMaxMsgSize())
+ || (itemMsglen > DataStoreUtils.MAX_MSG_DATA_STORE_SIZE)
|| (itemNext > totalBytes)) {
next = -1;
break;
@@ -438,7 +437,7 @@ public class FileSegment implements Segment {
if ((itemMsgPartId < 0)
|| (itemMsgOffset < 0)
|| (itemMsglen <= 0)
- || (itemMsglen > ClusterConfigHolder.getMaxMsgStoreLength())
+ || (itemMsglen > DataStoreUtils.STORE_MAX_MESSAGE_STORE_LEN)
|| (itemNext > totalBytes)) {
next = -1;
break;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java
index fe23dd3..400b666 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/msgstore/disk/MsgFileStore.java
@@ -35,7 +35,6 @@ import org.apache.tubemq.corebase.TErrCodeConstants;
import org.apache.tubemq.corebase.protobuf.generated.ClientBroker;
import org.apache.tubemq.corebase.utils.ServiceStatusHolder;
import org.apache.tubemq.server.broker.BrokerConfig;
-import org.apache.tubemq.server.broker.metadata.ClusterConfigHolder;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.stats.CountItem;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
@@ -274,7 +273,7 @@ public class MsgFileStore implements Closeable {
// skip when mismatch condition
if (curIndexDataOffset < 0
|| curIndexDataSize <= 0
- || curIndexDataSize > ClusterConfigHolder.getMaxMsgStoreLength()
+ || curIndexDataSize > DataStoreUtils.STORE_MAX_MESSAGE_STORE_LEN
|| curIndexDataOffset < curDataMinOffset) {
readedOffset = curIndexOffset + DataStoreUtils.STORE_INDEX_HEAD_LEN;
continue;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/DataStoreUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/DataStoreUtils.java
index 6f0fa13..6d79bf4 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/DataStoreUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/utils/DataStoreUtils.java
@@ -49,9 +49,6 @@ public class DataStoreUtils {
// + data 0
//
public static final int MAX_MSG_TRANSFER_SIZE = 1024 * 1024;
- public static final int MAX_MSG_DATA_STORE_SIZE =
- TBaseConstants.META_MAX_MESSAGE_DATA_SIZE * 2;
- public static final int MAX_READ_BUFFER_ADJUST = MAX_MSG_DATA_STORE_SIZE * 10;
public static final int STORE_DATA_PREFX_LEN = 48;
public static final int STORE_DATA_HEADER_LEN = STORE_DATA_PREFX_LEN + 4;
@@ -85,7 +82,9 @@ public class DataStoreUtils {
public static final int INDEX_POS_KEY_CODE = 16;
public static final int INDEX_POS_TIME_RECV = 20;
-
+ public static final int MAX_MSG_DATA_STORE_SIZE =
+ TBaseConstants.META_MAX_MESSAGE_DATA_SIZE_UPPER_LIMIT
+ + TBaseConstants.META_MB_UNIT_SIZE * 8;
public static final int STORE_MAX_MESSAGE_STORE_LEN
= STORE_DATA_HEADER_LEN + MAX_MSG_DATA_STORE_SIZE;
[incubator-tubemq] 21/49: [TUBEMQ-475] add the offset clone api of
the consume group
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit cc5796a424b7fde4d51d562eed39093550526379
Author: gosonzhang <go...@tencent.com>
AuthorDate: Fri Dec 25 10:40:51 2020 +0800
[TUBEMQ-475] add the offset clone api of the consume group
---
.../broker/metadata/BrokerMetadataManager.java | 3 +-
.../server/broker/metadata/MetadataManager.java | 2 +
.../server/broker/metadata/TopicMetadata.java | 16 ++
.../server/broker/offset/DefaultOffsetManager.java | 199 ++++++++++++++++-
.../tubemq/server/broker/offset/OffsetService.java | 18 ++
.../server/broker/web/BrokerAdminServlet.java | 246 +++++++++++++++++++++
.../tubemq/server/common/fielddef/WebFieldDef.java | 10 +-
.../server/common/offsetstorage/OffsetStorage.java | 11 +-
.../common/offsetstorage/ZkOffsetStorage.java | 139 +++++++++++-
.../common/offsetstorage/zookeeper/ZKUtil.java | 19 ++
.../org/apache/tubemq/server/master/TMaster.java | 3 +-
11 files changed, 647 insertions(+), 19 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/BrokerMetadataManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/BrokerMetadataManager.java
index 1e2f21e..8568372 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/BrokerMetadataManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/BrokerMetadataManager.java
@@ -143,7 +143,8 @@ public class BrokerMetadataManager implements MetadataManager {
return topicConfigMap.get(topic);
}
- public ConcurrentHashMap<String, TopicMetadata> getTopicConfigMap() {
+ @Override
+ public Map<String, TopicMetadata> getTopicConfigMap() {
return topicConfigMap;
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/MetadataManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/MetadataManager.java
index d638e5a..9ee9936 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/MetadataManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/MetadataManager.java
@@ -80,4 +80,6 @@ public interface MetadataManager {
String getDefDeletePolicy();
String getTopicDeletePolicy(String topic);
+
+ Map<String, TopicMetadata> getTopicConfigMap();
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
index 4ebfa4d..c582606 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/metadata/TopicMetadata.java
@@ -17,10 +17,15 @@
package org.apache.tubemq.server.broker.metadata;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.common.TStatusConstants;
+
/***
* Topic's metadata. Contains topic name, partitions count, etc.
*/
@@ -235,6 +240,17 @@ public class TopicMetadata {
this.unflushInterval = unflushInterval;
}
+ // builder the partitionId set for each store
+ public Set<Integer> getAllPartitionIds() {
+ Set<Integer> partIds = new HashSet<>();
+ for (int i = 0; i < numTopicStores; i++) {
+ for (int j = 0; j < numPartitions; j++) {
+ partIds.add(i * TBaseConstants.META_STORE_INS_BASE + j);
+ }
+ }
+ return partIds;
+ }
+
public int getStatusId() {
return statusId;
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
index 82a758c..bdd85b3 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
@@ -17,13 +17,17 @@
package org.apache.tubemq.server.broker.offset;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.daemon.AbstractDaemonService;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.broker.BrokerConfig;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
+import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorage;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorageInfo;
@@ -49,7 +53,8 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
public DefaultOffsetManager(final BrokerConfig brokerConfig) {
super("[Offset Manager]", brokerConfig.getZkConfig().getZkCommitPeriodMs());
this.brokerConfig = brokerConfig;
- zkOffsetStorage = new ZkOffsetStorage(brokerConfig.getZkConfig());
+ zkOffsetStorage = new ZkOffsetStorage(brokerConfig.getZkConfig(),
+ true, brokerConfig.getBrokerId());
super.start();
}
@@ -323,6 +328,187 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
}
/***
+ * Get in-memory and in zk group set
+ *
+ * @return booked group in memory and in zk
+ */
+ @Override
+ public Set<String> getBookedGroups() {
+ Set<String> groupSet = new HashSet<>();
+ groupSet.addAll(cfmOffsetMap.keySet());
+ Map<String, Set<String>> localGroups =
+ zkOffsetStorage.getZkLocalGroupTopicInfos();
+ groupSet.addAll(localGroups.keySet());
+ return groupSet;
+ }
+
+ /***
+ * Get in-memory group set
+ *
+ * @return booked group in memory
+ */
+ public Set<String> getInMemoryGroups() {
+ Set<String> cacheGroup = new HashSet<>();
+ cacheGroup.addAll(cfmOffsetMap.keySet());
+ return cacheGroup;
+ }
+
+ /***
+ * Get in-zookeeper but not in memory's group set
+ *
+ * @return booked group in zookeeper
+ */
+ @Override
+ public Set<String> getUnusedGroupInfo() {
+ Set<String> unUsedGroups = new HashSet<>();
+ Map<String, Set<String>> localGroups =
+ zkOffsetStorage.getZkLocalGroupTopicInfos();
+ for (String groupName : localGroups.keySet()) {
+ if (!cfmOffsetMap.containsKey(groupName)) {
+ unUsedGroups.add(groupName);
+ }
+ }
+ return unUsedGroups;
+ }
+
+ /***
+ * Get the topic set subscribed by the consumer group
+ * @param group
+ * @return topic set subscribed
+ */
+ @Override
+ public Set<String> getGroupSubInfo(String group) {
+ Set<String> result = new HashSet<>();
+ Map<String, OffsetStorageInfo> topicPartOffsetMap = cfmOffsetMap.get(group);
+ if (topicPartOffsetMap == null) {
+ Map<String, Set<String>> localGroups =
+ zkOffsetStorage.getZkLocalGroupTopicInfos();
+ result = localGroups.get(group);
+ } else {
+ for (OffsetStorageInfo storageInfo : topicPartOffsetMap.values()) {
+ result.add(storageInfo.getTopic());
+ }
+ }
+ return result;
+ }
+
+ /***
+ * Get group's offset by Specified topic-partitions
+ * @param group
+ * @param topicPartMap
+ * @return group offset info in memory or zk
+ */
+ @Override
+ public Map<String, Map<Integer, Long>> queryGroupOffset(
+ String group, Map<String, Set<Integer>> topicPartMap) {
+ Map<String, Map<Integer, Long>> result = new HashMap<>();
+ // search group from memory
+ Map<String, OffsetStorageInfo> topicPartOffsetMap = cfmOffsetMap.get(group);
+ if (topicPartOffsetMap == null) {
+ // query from zookeeper
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ Map<Integer, Long> qryResult =
+ zkOffsetStorage.queryGroupOffsetInfo(
+ group, entry.getKey(), entry.getValue());
+ Map<Integer, Long> offsetMap = new HashMap<>();
+ for (Map.Entry<Integer, Long> item : qryResult.entrySet()) {
+ if (item.getValue() != null) {
+ offsetMap.put(item.getKey(), item.getValue());
+ }
+ }
+ if (!offsetMap.isEmpty()) {
+ result.put(entry.getKey(), offsetMap);
+ }
+ }
+ } else {
+ // found in memory, get offset values
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ Map<Integer, Long> offsetMap = new HashMap<>();
+ for (Integer partitionId : entry.getValue()) {
+ String offsetCacheKey =
+ getOffsetCacheKey(entry.getKey(), partitionId);
+ OffsetStorageInfo offsetInfo = topicPartOffsetMap.get(offsetCacheKey);
+ if (offsetInfo != null) {
+ offsetMap.put(partitionId, offsetInfo.getOffset());
+ }
+ }
+ if (!offsetMap.isEmpty()) {
+ result.put(entry.getKey(), offsetMap);
+ }
+ }
+ }
+ return result;
+ }
+
+
+ /***
+ * Reset offset.
+ *
+ * @param storeManager
+ * @param groups
+ * @param topicPartOffsetMap
+ * @param modifier
+ * @return at least one record modified
+ */
+ @Override
+ public boolean modifyGroupOffset(MessageStoreManager storeManager, Set<String> groups,
+ Map<String, Map<Integer, Long>> topicPartOffsetMap,
+ String modifier) {
+ long oldOffset = -1;
+ long reSetOffset = -1;
+ boolean changed = false;
+ MessageStore store = null;
+ StringBuilder strBuidler = new StringBuilder(512);
+ // set offset by group
+ for (String group : groups) {
+ for (Map.Entry<String, Map<Integer, Long>> entry : topicPartOffsetMap.entrySet()) {
+ Map<Integer, Long> partOffsetMap = entry.getValue();
+ if (partOffsetMap == null) {
+ continue;
+ }
+ // set offset
+ for (Map.Entry<Integer, Long> entry1 : partOffsetMap.entrySet()) {
+ if (entry1.getValue() == null) {
+ continue;
+ }
+ reSetOffset = entry1.getValue();
+ // get topic store
+ try {
+ store = storeManager.getOrCreateMessageStore(
+ entry.getKey(), entry1.getKey());
+ } catch (Throwable e) {
+ //
+ }
+ if (store == null) {
+ continue;
+ }
+ long firstOffset = store.getIndexMinOffset();
+ long lastOffset = store.getIndexMaxOffset();
+ // adjust reseted offset value
+ reSetOffset = reSetOffset < firstOffset
+ ? firstOffset : Math.min(reSetOffset, lastOffset);
+ String offsetCacheKey =
+ getOffsetCacheKey(entry.getKey(), entry1.getKey());
+ getAndResetTmpOffset(group, offsetCacheKey);
+ OffsetStorageInfo regInfo = loadOrCreateOffset(group,
+ entry.getKey(), entry1.getKey(), offsetCacheKey, 0);
+ oldOffset = regInfo.getAndSetOffset(reSetOffset);
+ changed = true;
+ logger.info(strBuidler
+ .append("[Offset Manager] Update offset by modifier=")
+ .append(modifier).append(",reset offset=").append(reSetOffset)
+ .append(",old offset=").append(oldOffset)
+ .append(",updated offset=").append(regInfo.getOffset())
+ .append(",group=").append(group)
+ .append(",topic-partId=").append(offsetCacheKey).toString());
+ strBuidler.delete(0, strBuidler.length());
+ }
+ }
+ }
+ return changed;
+ }
+
+ /***
* Set temp offset.
*
* @param group
@@ -425,10 +611,10 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
OffsetStorageInfo regInfo = regInfoMap.get(offsetCacheKey);
if (regInfo == null) {
OffsetStorageInfo tmpRegInfo =
- zkOffsetStorage.loadOffset(group, topic, brokerConfig.getBrokerId(), partitionId);
+ zkOffsetStorage.loadOffset(group, topic, partitionId);
if (tmpRegInfo == null) {
- tmpRegInfo =
- new OffsetStorageInfo(topic, brokerConfig.getBrokerId(), partitionId, defOffset, 0);
+ tmpRegInfo = new OffsetStorageInfo(topic,
+ brokerConfig.getBrokerId(), partitionId, defOffset, 0);
}
regInfo = regInfoMap.putIfAbsent(offsetCacheKey, tmpRegInfo);
if (regInfo == null) {
@@ -443,4 +629,9 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
.append("-").append(partitionId).toString();
}
+ private String getOffsetCacheKey(String topic, String partitionId) {
+ return new StringBuilder(256).append(topic)
+ .append("-").append(partitionId).toString();
+ }
+
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
index 066fb3b..05f0724 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
@@ -17,9 +17,13 @@
package org.apache.tubemq.server.broker.offset;
+import java.util.Map;
+import java.util.Set;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
+import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorageInfo;
+
/***
* Offset manager service interface.
*/
@@ -51,4 +55,18 @@ public interface OffsetService {
long getTmpOffset(final String group, final String topic, int partitionId);
+ Set<String> getBookedGroups();
+
+ Set<String> getInMemoryGroups();
+
+ Set<String> getUnusedGroupInfo();
+
+ Set<String> getGroupSubInfo(String group);
+
+ Map<String, Map<Integer, Long>> queryGroupOffset(
+ String group, Map<String, Set<Integer>> topicPartMap);
+
+ boolean modifyGroupOffset(MessageStoreManager storeManager, Set<String> groups,
+ Map<String, Map<Integer, Long>> topicPartOffsetMap,
+ String modifier);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index 1b3f524..91bfd23 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -17,6 +17,8 @@
package org.apache.tubemq.server.broker.web;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -26,6 +28,7 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.broker.TubeBroker;
+import org.apache.tubemq.server.broker.metadata.TopicMetadata;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.nodeinfo.ConsumerNodeInfo;
@@ -71,6 +74,15 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// get all registered methods
innRegisterWebMethod("admin_get_methods",
"adminQueryAllMethods");
+ // Query all consumer groups booked on the Broker.
+ innRegisterWebMethod("admin_query_group",
+ "adminQueryBookedGroup");
+ // query consumer group's offset
+ innRegisterWebMethod("admin_query_offset",
+ "adminQueryGroupOffSet");
+ // clone consumer group's offset from source to target
+ innRegisterWebMethod("admin_clone_offset",
+ "adminCloneGroupOffSet");
}
public void adminQueryAllMethods(HttpServletRequest req,
@@ -560,5 +572,239 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("],\"totalCnt\":").append(totalCnt).append("}");
}
+ /***
+ * Query all consumer groups booked on the Broker.
+ *
+ * @param req
+ * @param sBuilder process result
+ */
+ public void adminQueryBookedGroup(HttpServletRequest req,
+ StringBuilder sBuilder) {
+ // get group list
+ ProcessResult result = WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.WITHDIVIDE, false, false);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ boolean withDivide = (boolean) result.retData1;
+ // get offset service
+ int itemCnt = 0;
+ int totalCnt = 0;
+ OffsetService offsetService = broker.getOffsetManager();
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
+ if (withDivide) {
+ // query in-memory group name set
+ Set<String> onlineGroups = offsetService.getInMemoryGroups();
+ sBuilder.append("{\"type\":\"in-cache\",\"groupName\":[");
+ for (String group : onlineGroups) {
+ if (itemCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("\"").append(group).append("\"");
+ }
+ sBuilder.append("],\"groupCount\":").append(itemCnt).append("}");
+ totalCnt++;
+ sBuilder.append(",");
+ // query in-zk group name set
+ itemCnt = 0;
+ Set<String> onZKGroup = offsetService.getUnusedGroupInfo();
+ sBuilder.append("{\"type\":\"in-zk\",\"groupName\":[");
+ for (String group : onZKGroup) {
+ if (itemCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("\"").append(group).append("\"");
+ }
+ sBuilder.append("],\"groupCount\":").append(itemCnt).append("}");
+ totalCnt++;
+ } else {
+ Set<String> allGroups = offsetService.getBookedGroups();
+ sBuilder.append("{\"type\":\"all\",\"groupName\":[");
+ for (String group : allGroups) {
+ if (itemCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("\"").append(group).append("\"");
+ }
+ sBuilder.append("],\"groupCount\":").append(itemCnt).append("}");
+ totalCnt++;
+ }
+ sBuilder.append("],\"dataCount\":").append(totalCnt).append("}");
+ }
+
+ /***
+ * Query consumer group offset.
+ *
+ * @param req
+ * @param sBuilder process result
+ */
+ public void adminQueryGroupOffSet(HttpServletRequest req,
+ StringBuilder sBuilder) {
+ // get group list
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, false, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ // filter invalid groups
+ Set<String> qryGroupNameSet = new HashSet<>();
+ Set<String> inGroupNameSet = (Set<String>) result.retData1;
+ Set<String> bookedGroupSet = broker.getOffsetManager().getBookedGroups();
+ if (inGroupNameSet.isEmpty()) {
+ qryGroupNameSet = bookedGroupSet;
+ } else {
+ for (String group : inGroupNameSet) {
+ if (bookedGroupSet.contains(group)) {
+ qryGroupNameSet.add(group);
+ }
+ }
+ }
+ // get the topic set to be queried
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ // get target consume group name
+ Set<String> topicSet = (Set<String>) result.retData1;
+ // verify the acquired Topic set and
+ // query the corresponding offset information
+ Map<String, Map<String, Map<Integer, Long>>> groupOffsetMaps = new HashMap<>();
+ for (String group : qryGroupNameSet) {
+ Map<String, Set<Integer>> topicPartMap =
+ validAndGetPartitions(group, topicSet);
+ Map<String, Map<Integer, Long>> groupOffsetMap =
+ broker.getOffsetManager().queryGroupOffset(group, topicPartMap);
+ groupOffsetMaps.put(group, groupOffsetMap);
+ }
+ // builder result
+ int totalCnt = 0;
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
+ for (Map.Entry<String, Map<String, Map<Integer, Long>>> entry
+ : groupOffsetMaps.entrySet()) {
+ if (totalCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ Map<String, Map<Integer, Long>> topicPartMap = entry.getValue();
+ sBuilder.append("{\"groupName\":\"").append(entry.getKey())
+ .append("\",\"subInfo\":[");
+ int topicCnt = 0;
+ for (Map.Entry<String, Map<Integer, Long>> entry1 : topicPartMap.entrySet()) {
+ if (topicCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ Map<Integer, Long> partOffMap = entry1.getValue();
+ sBuilder.append("{\"topicName\":\"").append(entry1.getKey())
+ .append("\",\"offsets\":[");
+ int partCnt = 0;
+ for (Map.Entry<Integer, Long> entry2 : partOffMap.entrySet()) {
+ if (partCnt++ > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("{\"").append(this.broker.getTubeConfig().getBrokerId())
+ .append(TokenConstants.ATTR_SEP).append(entry1.getKey())
+ .append(TokenConstants.ATTR_SEP).append(entry2.getKey())
+ .append("\":").append(entry2.getValue()).append("}");
+ }
+ sBuilder.append("],\"partCount\":").append(partCnt).append("}");
+ }
+ sBuilder.append("],\"topicCount\":").append(topicCnt).append("}");
+ }
+ sBuilder.append("],\"totalCnt\":").append(totalCnt).append("}");
+ }
+
+ /***
+ * Clone consume group offset, clone A group's offset to other group.
+ *
+ * @param req
+ * @param sBuilder process result
+ */
+ public void adminCloneGroupOffSet(HttpServletRequest req,
+ StringBuilder sBuilder) {
+ // get source consume group name
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.SRCGROUPNAME, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ final String srcGroupName = (String) result.retData1;
+ // get modify user
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ final String modifier = (String) result.retData1;
+ // get source consume group's topic set cloned to target group
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ // get target consume group name
+ Set<String> srcTopicNameSet = (Set<String>) result.retData1;
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.TGTCOMPSGROUPNAME, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ Set<String> tgtGroupNameSet = (Set<String>) result.retData1;
+ // check sourceGroup if existed
+ Set<String> bookedGroups = broker.getOffsetManager().getBookedGroups();
+ if (!bookedGroups.contains(srcGroupName)) {
+ WebParameterUtils.buildFailResult(sBuilder,
+ new StringBuilder(512).append("Parameter ")
+ .append(WebFieldDef.SRCGROUPNAME.name).append(": ")
+ .append(srcGroupName)
+ .append(" has not been registered on this Broker!").toString());
+ return;
+ }
+ // valid topic and get topic's partitionIds
+ Map<String, Set<Integer>> topicPartMap =
+ validAndGetPartitions(srcGroupName, srcTopicNameSet);
+ if (topicPartMap.isEmpty()) {
+ WebParameterUtils.buildFailResult(sBuilder,
+ new StringBuilder(512).append("Parameter ")
+ .append(WebFieldDef.SRCGROUPNAME.name).append(": not found ")
+ .append(srcGroupName).append(" subscribed topic set!").toString());
+ return;
+ }
+ // query offset from source group
+ Map<String, Map<Integer, Long>> srcGroupOffsets =
+ broker.getOffsetManager().queryGroupOffset(srcGroupName, topicPartMap);
+ boolean changed = broker.getOffsetManager().modifyGroupOffset(
+ broker.getStoreManager(), tgtGroupNameSet, srcGroupOffsets, modifier);
+ // builder return result
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
+ }
+
+ private Map<String, Set<Integer>> validAndGetPartitions(String group, Set<String> topicSet) {
+ Map<String, Set<Integer>> topicPartMap = new HashMap<>();
+ // query stored topic set stored in memory or zk
+ if (topicSet.isEmpty()) {
+ topicSet = broker.getOffsetManager().getGroupSubInfo(group);
+ }
+ // get topic's partitionIds
+ if (topicSet != null) {
+ Map<String, TopicMetadata> topicConfigMap =
+ broker.getMetadataManager().getTopicConfigMap();
+ if (topicConfigMap != null) {
+ for (String topic : topicSet) {
+ TopicMetadata topicMetadata = topicConfigMap.get(topic);
+ if (topicMetadata != null) {
+ topicPartMap.put(topic, topicMetadata.getAllPartitionIds());
+ }
+ }
+ }
+ }
+ return topicPartMap;
+ }
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index f73959e..ec97421 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -70,7 +70,15 @@ public enum WebFieldDef {
COMPSBROKERID(15, "brokerId", "brokerId", WebFieldType.COMPINT,
"Broker ID", RegexDef.TMP_NUMBER),
WITHIP(16, "withIp", "ip", WebFieldType.BOOLEAN,
- "Require return ip information, default is false");
+ "Require return ip information, default is false"),
+ WITHDIVIDE(17, "divide", "div", WebFieldType.BOOLEAN,
+ "Need to divide the returned result, default is false"),
+ SRCGROUPNAME(18, "sourceGroupName", "srcGroup", WebFieldType.STRING,
+ "Offset clone source group name", TBaseConstants.META_MAX_GROUPNAME_LENGTH,
+ RegexDef.TMP_GROUP),
+ TGTCOMPSGROUPNAME(19, "targetGroupName", "tgtGroup",
+ WebFieldType.COMPSTRING, "Offset clone target group name",
+ TBaseConstants.META_MAX_GROUPNAME_LENGTH, RegexDef.TMP_GROUP);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java
index 470aafd..dca7ca8 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/OffsetStorage.java
@@ -18,6 +18,8 @@
package org.apache.tubemq.server.common.offsetstorage;
import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
public interface OffsetStorage {
@@ -25,9 +27,16 @@ public interface OffsetStorage {
void close();
OffsetStorageInfo loadOffset(final String group,
- final String topic, int brokerId, int partitionId);
+ final String topic, int partitionId);
void commitOffset(final String group,
final Collection<OffsetStorageInfo> offsetInfoList,
boolean isFailRetry);
+
+ Map<String, Map<String, Set<String>>> getZkGroupTopicBrokerInfos();
+
+ Map<String, Set<String>> getZkLocalGroupTopicInfos();
+
+ Map<Integer, Long> queryGroupOffsetInfo(String group, String topic,
+ Set<Integer> partitionIds);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java
index 7c39bfd..8094151 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/ZkOffsetStorage.java
@@ -19,6 +19,12 @@ package org.apache.tubemq.server.common.offsetstorage;
import java.net.BindException;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.server.broker.exception.OffsetStoreException;
import org.apache.tubemq.server.common.TServerConstants;
@@ -42,7 +48,7 @@ public class ZkOffsetStorage implements OffsetStorage {
public void uncaughtException(Thread t, Throwable e) {
if (e instanceof BindException) {
logger.error("Bind failed.", e);
- System.exit(1);
+ // System.exit(1);
}
if (e instanceof IllegalStateException
&& e.getMessage().contains("Shutdown in progress")) {
@@ -56,27 +62,33 @@ public class ZkOffsetStorage implements OffsetStorage {
private final String tubeZkRoot;
private final String consumerZkDir;
+ private final boolean isBroker;
+ private final int brokerId;
private ZKConfig zkConfig;
private ZooKeeperWatcher zkw;
+ // group-topic-brokerid
+ private final Map<String, Map<String, Set<String>>> zkGroupTopicBrokerInfos = new HashMap<>();
+ // group-topic
+ private final Map<String, Set<String>> zkLocalGroupTopicInfos = new HashMap<>();
- /**
- * Constructor of ZkOffsetStorage
- *
- * @param zkConfig the zookeeper configuration
- */
- public ZkOffsetStorage(final ZKConfig zkConfig) {
+
+ public ZkOffsetStorage(final ZKConfig zkConfig, boolean isBroker, int brokerId) {
this.zkConfig = zkConfig;
+ this.brokerId = brokerId;
+ this.isBroker = isBroker;
this.tubeZkRoot = normalize(this.zkConfig.getZkNodeRoot());
this.consumerZkDir = this.tubeZkRoot + "/consumers-v3";
try {
this.zkw = new ZooKeeperWatcher(zkConfig);
} catch (Throwable e) {
logger.error(new StringBuilder(256)
- .append("Failed to connect ZooKeeper server (")
+ .append("[ZkOffsetStorage] Failed to connect ZooKeeper server (")
.append(this.zkConfig.getZkServerAddr()).append(") !").toString(), e);
System.exit(1);
}
- logger.info("ZooKeeper Offset Storage initiated!");
+ logger.info("[ZkOffsetStorage] Get group-topic-broker info from ZooKeeper");
+ queryAllZKGroupTopicInfo();
+ logger.info("[ZkOffsetStorage] ZooKeeper Offset Storage initiated!");
}
@Override
@@ -90,6 +102,16 @@ public class ZkOffsetStorage implements OffsetStorage {
}
@Override
+ public Map<String, Map<String, Set<String>>> getZkGroupTopicBrokerInfos() {
+ return zkGroupTopicBrokerInfos;
+ }
+
+ @Override
+ public Map<String, Set<String>> getZkLocalGroupTopicInfos() {
+ return zkLocalGroupTopicInfos;
+ }
+
+ @Override
public void commitOffset(final String group,
final Collection<OffsetStorageInfo> offsetInfoList,
boolean isFailRetry) {
@@ -125,10 +147,11 @@ public class ZkOffsetStorage implements OffsetStorage {
}
@Override
- public OffsetStorageInfo loadOffset(final String group, final String topic, int brokerId, int partitionId) {
+ public OffsetStorageInfo loadOffset(final String group, final String topic, int partitionId) {
String znode = new StringBuilder(512).append(this.consumerZkDir).append("/")
.append(group).append("/offsets/").append(topic).append("/")
- .append(brokerId).append(TokenConstants.HYPHEN).append(partitionId).toString();
+ .append(brokerId).append(TokenConstants.HYPHEN)
+ .append(partitionId).toString();
String offsetZkInfo;
try {
offsetZkInfo = ZKUtil.readDataMaybeNull(this.zkw, znode);
@@ -183,6 +206,100 @@ public class ZkOffsetStorage implements OffsetStorage {
}
}
+ /**
+ * Get offset stored in zookeeper, if not found or error, set null
+ * <p/>
+ *
+ * @return partitionId--offset map info
+ */
+ @Override
+ public Map<Integer, Long> queryGroupOffsetInfo(String group, String topic,
+ Set<Integer> partitionIds) {
+ StringBuilder sBuider = new StringBuilder(512);
+ String basePath = sBuider.append(this.consumerZkDir).append("/")
+ .append(group).append("/offsets/").append(topic).append("/")
+ .append(brokerId).append(TokenConstants.HYPHEN).toString();
+ sBuider.delete(0, sBuider.length());
+ String offsetZkInfo = null;
+ Map<Integer, Long> offsetMap = new HashMap<>(partitionIds.size());
+ for (Integer partitionId : partitionIds) {
+ String offsetNode = sBuider.append(basePath).append(partitionId).toString();
+ sBuider.delete(0, sBuider.length());
+ try {
+ offsetZkInfo = ZKUtil.readDataMaybeNull(this.zkw, offsetNode);
+ if (offsetZkInfo == null) {
+ offsetMap.put(partitionId, null);
+ } else {
+ String[] offsetInfoStrs =
+ offsetZkInfo.split(TokenConstants.HYPHEN);
+ offsetMap.put(partitionId, Long.parseLong(offsetInfoStrs[1]));
+ }
+ } catch (Throwable e) {
+ offsetMap.put(partitionId, null);
+ }
+ }
+ return offsetMap;
+ }
+
+ /**
+ * Get group-topic-brokerid map info stored in zookeeper.
+ * <p/>
+ * The broker only cares about the content of its own node,
+ * so this part only queries when the node starts, and
+ * caches relevant data in the memory for finding
+ *
+ */
+ private void queryAllZKGroupTopicInfo() {
+ StringBuilder sBuider = new StringBuilder(512);
+ // get all booked groups name
+ String groupNode = sBuider.append(this.consumerZkDir).toString();
+ List<String> bookedGroups = ZKUtil.getChildren(this.zkw, groupNode);
+ sBuider.delete(0, sBuider.length());
+ if (bookedGroups != null) {
+ // get topic info by group
+ for (String group : bookedGroups) {
+ String topicNode = sBuider.append(groupNode)
+ .append("/").append(group).append("/offsets").toString();
+ List<String> consumeTopics = ZKUtil.getChildren(this.zkw, topicNode);
+ sBuider.delete(0, sBuider.length());
+ Set<String> topicSet = new HashSet<>();
+ Map<String, Set<String>> topicBrokerSet = new HashMap<>();
+ if (consumeTopics != null) {
+ // get broker info by topic
+ for (String topic : consumeTopics) {
+ String brokerNode = sBuider.append(topicNode)
+ .append("/").append(topic).toString();
+ List<String> brokerIds = ZKUtil.getChildren(this.zkw, brokerNode);
+ sBuider.delete(0, sBuider.length());
+ Set<String> brokerIdSet = new HashSet<>();
+ if (brokerIds != null) {
+ for (String idStr : brokerIds) {
+ if (idStr != null) {
+ String[] brokerPartIdStrs =
+ idStr.split(TokenConstants.HYPHEN);
+ brokerIdSet.add(brokerPartIdStrs[0]);
+ }
+ }
+ if (isBroker && brokerIdSet.contains(String.valueOf(brokerId))) {
+ topicSet.add(topic);
+ }
+ }
+ topicBrokerSet.put(topic, brokerIdSet);
+ }
+ }
+ if (!topicSet.isEmpty()) {
+ zkLocalGroupTopicInfos.put(group, topicSet);
+ }
+ zkGroupTopicBrokerInfos.put(group, topicBrokerSet);
+ }
+ }
+ logger.info(new StringBuilder(256)
+ .append("[ZkOffsetStorage] query from zookeeper, total group size = ")
+ .append(zkGroupTopicBrokerInfos.size()).append(", local group size = ")
+ .append(zkLocalGroupTopicInfos.size()).toString());
+ }
+
+
private String normalize(final String root) {
if (root.startsWith("/")) {
return this.removeLastSlash(root);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java
index 77d5436..da86191 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/offsetstorage/zookeeper/ZKUtil.java
@@ -19,6 +19,8 @@ package org.apache.tubemq.server.common.offsetstorage.zookeeper;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
+
import org.apache.commons.codec.binary.StringUtils;
import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.server.common.fileconfig.ZKConfig;
@@ -147,6 +149,23 @@ public class ZKUtil {
return getDataInternal(zkw, znode, null, true);
}
+ /**
+ * Get the children data at the specified znode.
+ * <p/>
+ * Returns the children data. Returns null if
+ * the node does not exist or there is an exception.
+ *
+ * @param zkw zk reference
+ * @param znode path of node
+ * @return children data of the specified znode, or null
+ */
+ public static List<String> getChildren(ZooKeeperWatcher zkw, String znode) {
+ try {
+ return zkw.getRecoverableZooKeeper().getChildren(znode, false);
+ } catch (Throwable e) {
+ return null;
+ }
+ }
private static byte[] getDataInternal(ZooKeeperWatcher zkw, String znode, Stat stat,
boolean watcherSet) throws KeeperException {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
index 34570de..cc6f681 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
@@ -176,7 +176,8 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
this.consumerEventManager = new ConsumerEventManager(consumerHolder);
this.topicPSInfoManager = new TopicPSInfoManager();
this.loadBalancer = new DefaultLoadBalancer();
- this.zkOffsetStorage = new ZkOffsetStorage(this.masterConfig.getZkConfig());
+ this.zkOffsetStorage = new ZkOffsetStorage(this.masterConfig.getZkConfig(),
+ false, TBaseConstants.META_VALUE_UNDEFINED);
this.heartbeatManager = new HeartbeatManager();
heartbeatManager.regConsumerCheckBusiness(masterConfig.getConsumerHeartbeatTimeoutMs(),
new TimeoutListener() {
[incubator-tubemq] 19/49: [TUBEMQ-470] Add query API of TopicName
and BrokerId collection
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 8e0b01de8aee18f7d4899fc772a4d809a915f9fa
Author: gosonzhang <go...@tencent.com>
AuthorDate: Tue Dec 22 18:59:39 2020 +0800
[TUBEMQ-470] Add query API of TopicName and BrokerId collection
---
.../tubemq/server/common/fielddef/WebFieldDef.java | 10 ++-
.../server/common/utils/WebParameterUtils.java | 53 ++++++++++-
.../nodemanage/nodebroker/BrokerConfManager.java | 69 ++++++++++++++
.../web/handler/WebBrokerTopicConfHandler.java | 100 +++++++++++++++++++++
4 files changed, 226 insertions(+), 6 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index 44a6f81..f73959e 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -64,9 +64,13 @@ public enum WebFieldDef {
COMPSPARTITIONID(12, "partitionId", "pid", WebFieldType.COMPINT,
"Partition id", RegexDef.TMP_NUMBER),
CALLERIP(13, "callerIp", "cip", WebFieldType.STRING,
- "Caller ip address", TBaseConstants.META_MAX_CLIENT_HOSTNAME_LENGTH);
-
-
+ "Caller ip address", TBaseConstants.META_MAX_CLIENT_HOSTNAME_LENGTH),
+ BROKERID(14, "brokerId", "brokerId", WebFieldType.INT,
+ "Broker ID", RegexDef.TMP_NUMBER),
+ COMPSBROKERID(15, "brokerId", "brokerId", WebFieldType.COMPINT,
+ "Broker ID", RegexDef.TMP_NUMBER),
+ WITHIP(16, "withIp", "ip", WebFieldType.BOOLEAN,
+ "Require return ip information, default is false");
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index b7d5d41..1202d33 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -283,6 +283,51 @@ public class WebParameterUtils {
* @param req Http Servlet Request
* @param fieldDef the parameter field definition
* @param required a boolean value represent whether the parameter is must required
+ * @return valid result for the parameter value
+ */
+ public static ProcessResult getIntParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required) {
+ ProcessResult procResult =
+ getStringParamValue(req, fieldDef, required, null);
+ if (!procResult.success) {
+ return procResult;
+ }
+ ProcessResult procRet = new ProcessResult();
+ Set<Integer> tgtValueSet = new HashSet<Integer>();
+ if (fieldDef.isCompFieldType()) {
+ Set<String> valItemSet = (Set<String>) procResult.retData1;
+ if (valItemSet.isEmpty()) {
+ procResult.setSuccResult(tgtValueSet);
+ return procResult;
+ }
+ for (String itemVal : valItemSet) {
+ if (!checkIntValueNorms(procRet, fieldDef, itemVal, false, -1)) {
+ return procRet;
+ }
+ tgtValueSet.add((Integer) procRet.retData1);
+ }
+ } else {
+ String paramValue = (String) procResult.retData1;
+ if (paramValue == null) {
+ procResult.setSuccResult(tgtValueSet);
+ return procResult;
+ }
+ if (!checkIntValueNorms(procRet,
+ fieldDef, paramValue, false, -1)) {
+ tgtValueSet.add((Integer) procRet.retData1);
+ }
+ }
+ procResult.setSuccResult(tgtValueSet);
+ return procResult;
+ }
+
+ /**
+ * Parse the parameter value from an object value to a integer value
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
* @param defValue a default value returned if failed to parse value from the given object
* @param minValue min value required
* @return valid result for the parameter value
@@ -307,7 +352,7 @@ public class WebParameterUtils {
}
ProcessResult procRet = new ProcessResult();
for (String itemVal : valItemSet) {
- if (!checkIntValueNorms(procRet, fieldDef, itemVal, minValue)) {
+ if (!checkIntValueNorms(procRet, fieldDef, itemVal, true, minValue)) {
return procRet;
}
tgtValueSet.add((Integer) procRet.retData1);
@@ -319,7 +364,7 @@ public class WebParameterUtils {
procResult.setSuccResult(defValue);
return procResult;
}
- checkIntValueNorms(procResult, fieldDef, paramValue, minValue);
+ checkIntValueNorms(procResult, fieldDef, paramValue, true, minValue);
}
return procResult;
}
@@ -501,16 +546,18 @@ public class WebParameterUtils {
* @param procResult process result
* @param fieldDef the parameter field definition
* @param paramValue the parameter value
+ * @param hasMinVal whether there is a minimum
* param minValue the parameter min value
* @return check result for string value of parameter
*/
private static boolean checkIntValueNorms(ProcessResult procResult,
WebFieldDef fieldDef,
String paramValue,
+ boolean hasMinVal,
int minValue) {
try {
int paramIntVal = Integer.parseInt(paramValue);
- if (paramIntVal < minValue) {
+ if (hasMinVal && paramIntVal < minValue) {
procResult.setFailResult(400,
new StringBuilder(512).append("Parameter ")
.append(fieldDef.name).append(" value must >= ")
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
index 61e1fe3..698f0d3 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/nodemanage/nodebroker/BrokerConfManager.java
@@ -721,6 +721,75 @@ public class BrokerConfManager implements Server {
return true;
}
+ /**
+ * get broker's topicName set,
+ * if brokerIds is empty, then return all broker's topicNames
+ *
+ * @param brokerIdSet
+ * @return broker's topicName set
+ */
+ public Map<Integer, Set<String>> getBrokerTopicConfigInfo(Set<Integer> brokerIdSet) {
+ Map<Integer, Set<String>> result = new HashMap<>();
+ if (brokerIdSet.isEmpty()) {
+ for (ConcurrentHashMap.Entry<Integer,
+ ConcurrentHashMap<String, BdbTopicConfEntity>>
+ entry : brokerTopicEntityStoreMap.entrySet()) {
+ Set<String> topicSet = new HashSet<>();
+ if (entry.getValue() != null) {
+ topicSet.addAll(entry.getValue().keySet());
+ }
+ result.put(entry.getKey(), topicSet);
+ }
+ } else {
+ for (Integer brokerId : brokerIdSet) {
+ ConcurrentHashMap<String, BdbTopicConfEntity> topicConfigMap =
+ brokerTopicEntityStoreMap.get(brokerId);
+ Set<String> topicSet = new HashSet<>();
+ if (topicConfigMap != null && !topicConfigMap.isEmpty()) {
+ topicSet.addAll(topicConfigMap.keySet());
+ }
+ result.put(brokerId, topicSet);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * get topic's brokerId set,
+ * if topicSet is empty, then return all topic's brokerIds
+ *
+ * @param topicNameSet
+ * @return topic's brokerId set
+ */
+ public Map<String, Map<Integer, String>> getTopicBrokerConfigInfo(Set<String> topicNameSet) {
+ Map<String, Map<Integer, String>> result = new HashMap<>();
+ if (topicNameSet.isEmpty()) {
+ for (Map<String, BdbTopicConfEntity> topicConfigMap
+ : brokerTopicEntityStoreMap.values()) {
+ for (Map.Entry<String, BdbTopicConfEntity> entry
+ : topicConfigMap.entrySet()) {
+ Map<Integer, String> brokerInfos =
+ result.computeIfAbsent(entry.getKey(), k -> new HashMap<>());
+ brokerInfos.put(entry.getValue().getBrokerId(),
+ entry.getValue().getBrokerIp());
+ }
+ }
+ } else {
+ for (Map<String, BdbTopicConfEntity> topicConfigMap
+ : brokerTopicEntityStoreMap.values()) {
+ for (String topic : topicNameSet) {
+ Map<Integer, String> brokerInfos =
+ result.computeIfAbsent(topic, k -> new HashMap<>());
+ BdbTopicConfEntity topicConfig = topicConfigMap.get(topic);
+ if (topicConfig != null) {
+ brokerInfos.put(topicConfig.getBrokerId(), topicConfig.getBrokerIp());
+ }
+ }
+ }
+ }
+ return result;
+ }
+
public Set<String> getTotalConfiguredTopicNames() {
Set<String> totalTopics = new HashSet<>(50);
for (ConcurrentHashMap<String, BdbTopicConfEntity> tmpTopicCfgMap
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
index bc14775..3bdfe16 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/handler/WebBrokerTopicConfHandler.java
@@ -33,6 +33,8 @@ import org.apache.tubemq.corebase.cluster.TopicInfo;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.server.common.TServerConstants;
import org.apache.tubemq.server.common.TStatusConstants;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
+import org.apache.tubemq.server.common.utils.ProcessResult;
import org.apache.tubemq.server.common.utils.WebParameterUtils;
import org.apache.tubemq.server.master.TMaster;
import org.apache.tubemq.server.master.bdbstore.bdbentitys.BdbBrokerConfEntity;
@@ -66,6 +68,10 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
"adminQueryTopicCfgEntityAndRunInfo");
registerQueryWebMethod("admin_query_broker_topic_config_info",
"adminQueryBrokerTopicCfgAndRunInfo");
+ registerQueryWebMethod("admin_query_topicName",
+ "adminQuerySimpleTopicName");
+ registerQueryWebMethod("admin_query_brokerId",
+ "adminQuerySimpleBrokerId");
// register modify method
registerModifyWebMethod("admin_add_new_topic_record",
"adminAddTopicEntityInfo");
@@ -702,6 +708,100 @@ public class WebBrokerTopicConfHandler extends AbstractWebHandler {
}
/**
+ * Query broker's topic-name set info
+ *
+ * @param req
+ * @return
+ */
+ public StringBuilder adminQuerySimpleTopicName(HttpServletRequest req) {
+ StringBuilder strBuffer = new StringBuilder(512);
+ ProcessResult result = WebParameterUtils.getIntParamValue(req,
+ WebFieldDef.COMPSBROKERID, false);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(strBuffer, result.errInfo);
+ return strBuffer;
+ }
+ Set<Integer> brokerIds = (Set<Integer>) result.retData1;
+ strBuffer.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\",\"data\":[");
+ Map<Integer, Set<String>> brokerTopicConfigMap =
+ brokerConfManager.getBrokerTopicConfigInfo(brokerIds);
+ int dataCount = 0;
+ for (Map.Entry<Integer, Set<String>> entry : brokerTopicConfigMap.entrySet()) {
+ if (dataCount++ > 0) {
+ strBuffer.append(",");
+ }
+ strBuffer.append("{\"brokerId\":").append(entry.getKey()).append(",\"topicName\":[");
+ int topicCnt = 0;
+ Set<String> topicSet = entry.getValue();
+ for (String topic : topicSet) {
+ if (topicCnt++ > 0) {
+ strBuffer.append(",");
+ }
+ strBuffer.append("\"").append(topic).append("\"");
+ }
+ strBuffer.append("],\"topicCount\":").append(topicCnt).append("}");
+ }
+ strBuffer.append("],\"dataCount\":").append(dataCount).append("}");
+ return strBuffer;
+ }
+
+ /**
+ * Query topic's broker id set
+ *
+ * @param req
+ * @return
+ */
+ public StringBuilder adminQuerySimpleBrokerId(HttpServletRequest req) {
+ StringBuilder strBuffer = new StringBuilder(512);
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, false, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(strBuffer, result.errInfo);
+ return strBuffer;
+ }
+ Set<String> topicNameSet = (Set<String>) result.retData1;
+ result = WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.WITHIP, false, false);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(strBuffer, result.errInfo);
+ return strBuffer;
+ }
+ boolean withIp = (Boolean) result.retData1;
+ // return result;
+ strBuffer.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\",\"data\":[");
+ Map<String, Map<Integer, String>> opicBrokerConfigMap =
+ brokerConfManager.getTopicBrokerConfigInfo(topicNameSet);
+ int dataCount = 0;
+ for (Map.Entry<String, Map<Integer, String>> entry : opicBrokerConfigMap.entrySet()) {
+ if (dataCount++ > 0) {
+ strBuffer.append(",");
+ }
+ strBuffer.append("{\"topicName\":\"").append(entry.getKey()).append("\",\"brokerInfo\":[");
+ int topicCnt = 0;
+ Map<Integer, String> brokerMap = entry.getValue();
+ if (withIp) {
+ for (Map.Entry<Integer, String> entry1 : brokerMap.entrySet()) {
+ if (topicCnt++ > 0) {
+ strBuffer.append(",");
+ }
+ strBuffer.append("{\"brokerId\":").append(entry1.getKey())
+ .append(",\"brokerIp\":\"").append(entry1.getValue()).append("\"}");
+ }
+ } else {
+ for (Map.Entry<Integer, String> entry1 : brokerMap.entrySet()) {
+ if (topicCnt++ > 0) {
+ strBuffer.append(",");
+ }
+ strBuffer.append(entry1.getKey());
+ }
+ }
+ strBuffer.append("],\"brokerCnt\":").append(topicCnt).append("}");
+ }
+ strBuffer.append("],\"dataCount\":").append(dataCount).append("}");
+ return strBuffer;
+ }
+
+ /**
* Delete topic info
*
* @param req
[incubator-tubemq] 31/49: [TUBEMQ-505] Remove the "WIP" label of
the DISCLAIMER file (#390)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 78ced32bdb264467ec82c28b22d1e2c38cd7c1b3
Author: gosonzhang <46...@qq.com>
AuthorDate: Mon Jan 11 10:18:30 2021 +0800
[TUBEMQ-505] Remove the "WIP" label of the DISCLAIMER file (#390)
Co-authored-by: gosonzhang <go...@tencent.com>
---
DISCLAIMER | 6 ++++++
DISCLAIMER-WIP | 18 ------------------
2 files changed, 6 insertions(+), 18 deletions(-)
diff --git a/DISCLAIMER b/DISCLAIMER
new file mode 100644
index 0000000..06b6195
--- /dev/null
+++ b/DISCLAIMER
@@ -0,0 +1,6 @@
+Apache TubeMQ is an effort undergoing incubation at The Apache Software Foundation (ASF),
+sponsored by the Apache Incubator. Incubation is required of all newly accepted projects
+until a further review indicates that the infrastructure, communications, and decision
+making process have stabilized in a manner consistent with other successful ASF projects.
+While incubation status is not necessarily a reflection of the completeness or stability
+of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
\ No newline at end of file
diff --git a/DISCLAIMER-WIP b/DISCLAIMER-WIP
deleted file mode 100644
index 6c72cf3..0000000
--- a/DISCLAIMER-WIP
+++ /dev/null
@@ -1,18 +0,0 @@
-Apache TubeMQ is an effort undergoing incubation at The Apache Software Foundation (ASF),
-sponsored by the Apache Incubator. Incubation is required of all newly accepted projects
-until a further review indicates that the infrastructure, communications, and decision
-making process have stabilized in a manner consistent with other successful ASF projects.
-While incubation status is not necessarily a reflection of the completeness or stability
-of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
-
-Some of the incubating project's releases may not be fully compliant with ASF policy. For
-example, releases may have incomplete or un-reviewed licensing conditions. What follows is
-a list of known issues the project is currently aware of (note that this list, by definition,
-is likely to be incomplete):
-
- * Releases may have incomplete licensing conditions
-
-If you are planning to incorporate this work into your product/project, please be aware that
-you will need to conduct a thorough licensing review to determine the overall implications of
-including this work. For the current status of this project through the Apache Incubator
-visit: https://incubator.apache.org/projects/tubemq.html
\ No newline at end of file
[incubator-tubemq] 09/49: [TUBEMQ-444]Add consume and produce Cli
commands (#343)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 6ce60a894b1bed29d8bd1602d936cf9c32581b77
Author: gosonzhang <46...@qq.com>
AuthorDate: Wed Dec 9 15:38:34 2020 +0800
[TUBEMQ-444]Add consume and produce Cli commands (#343)
Co-authored-by: gosonzhang <go...@tencent.com>
---
...nsumer-perf-test.sh => tubemq-consumer-test.sh} | 2 +-
...oducer-perf-test.sh => tubemq-producer-test.sh} | 2 +-
.../apache/tubemq/corebase/TErrCodeConstants.java | 6 +
.../apache/tubemq/corebase/utils/MixedUtils.java | 59 +++
tubemq-example/pom.xml | 10 -
tubemq-example/src/main/assembly/assembly.xml | 3 -
.../apache/tubemq/example/ArgsParserHelper.java | 48 ---
.../tubemq/example/MAMessageProducerExample.java | 100 ++----
.../tubemq/example/MessageConsumerExample.java | 125 +++----
.../tubemq/server/common/fielddef/CliArgDef.java | 70 ++--
.../tubemq/server/tools/cli/CliAbstractBase.java | 76 ++++
.../tubemq/server/tools/cli/CliConsumer.java | 394 +++++++++++++++++++++
.../tubemq/server/tools/cli/CliProducer.java | 385 ++++++++++++++++++++
13 files changed, 1037 insertions(+), 243 deletions(-)
diff --git a/bin/tubemq-consumer-perf-test.sh b/bin/tubemq-consumer-test.sh
similarity index 94%
rename from bin/tubemq-consumer-perf-test.sh
rename to bin/tubemq-consumer-test.sh
index 54189a2..23292fb 100644
--- a/bin/tubemq-consumer-perf-test.sh
+++ b/bin/tubemq-consumer-test.sh
@@ -37,4 +37,4 @@ if [ -z "$BASE_DIR" ] ; then
#echo "TubeMQ master is at $BASE_DIR"
fi
source $BASE_DIR/bin/env.sh
-$JAVA $TOOLS_ARGS org.apache.tubemq.example.MessageConsumerExample $@
+$JAVA $TOOLS_ARGS org.apache.tubemq.server.tools.cli.CliConsumer $@
diff --git a/bin/tubemq-producer-perf-test.sh b/bin/tubemq-producer-test.sh
similarity index 94%
rename from bin/tubemq-producer-perf-test.sh
rename to bin/tubemq-producer-test.sh
index ed76c3a..0f96033 100644
--- a/bin/tubemq-producer-perf-test.sh
+++ b/bin/tubemq-producer-test.sh
@@ -37,4 +37,4 @@ if [ -z "$BASE_DIR" ] ; then
#echo "TubeMQ master is at $BASE_DIR"
fi
source $BASE_DIR/bin/env.sh
-$JAVA $TOOLS_ARGS org.apache.tubemq.example.MAMessageProducerExample $@
+$JAVA $TOOLS_ARGS org.apache.tubemq.server.tools.cli.CliProducer $@
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/TErrCodeConstants.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/TErrCodeConstants.java
index 441af38..8bdafa0 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/TErrCodeConstants.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/TErrCodeConstants.java
@@ -18,6 +18,9 @@
package org.apache.tubemq.corebase;
+import java.util.Arrays;
+import java.util.List;
+
public class TErrCodeConstants {
public static final int SUCCESS = 200;
public static final int NOT_READY = 201;
@@ -44,4 +47,7 @@ public class TErrCodeConstants {
public static final int SERVICE_UNAVAILABLE = 503;
public static final int INTERNAL_SERVER_ERROR_MSGSET_NULL = 510;
+ public static final List<Integer> IGNORE_ERROR_SET =
+ Arrays.asList(BAD_REQUEST, NOT_FOUND, ALL_PARTITION_FROZEN,
+ NO_PARTITION_ASSIGNED, ALL_PARTITION_WAITING, ALL_PARTITION_INUSE);
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
index 17ccb5e..bfbedad 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/MixedUtils.java
@@ -17,6 +17,16 @@
package org.apache.tubemq.corebase.utils;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeSet;
+
+import org.apache.commons.codec.binary.StringUtils;
+import org.apache.tubemq.corebase.TokenConstants;
+
+
+
public class MixedUtils {
// java version cache
private static String javaVersion = "";
@@ -33,4 +43,53 @@ public class MixedUtils {
return javaVersion.substring(0, maxLen);
}
}
+
+ /**
+ * parse topic Parameter with format topic_1[,topic_2[:filterCond_2.1[;filterCond_2.2]]]
+ * topicParam->set(filterCond) map
+ * @param topicParam - composite string
+ * @return - map of topic->set(filterCond)
+ */
+ public static Map<String, TreeSet<String>> parseTopicParam(String topicParam) {
+ Map<String, TreeSet<String>> topicAndFiltersMap = new HashMap<>();
+ if (TStringUtils.isBlank(topicParam)) {
+ return topicAndFiltersMap;
+ }
+ String[] topicFilterStrs = topicParam.split(TokenConstants.ARRAY_SEP);
+ for (String topicFilterStr : topicFilterStrs) {
+ if (TStringUtils.isBlank(topicFilterStr)) {
+ continue;
+ }
+ String[] topicFilters = topicFilterStr.split(TokenConstants.ATTR_SEP);
+ if (TStringUtils.isBlank(topicFilters[0])) {
+ continue;
+ }
+ TreeSet<String> filterSet = new TreeSet<>();
+ if (topicFilters.length > 1
+ && TStringUtils.isNotBlank(topicFilters[1])) {
+ String[] filterItems = topicFilters[1].split(TokenConstants.LOG_SEG_SEP);
+ for (String filterItem : filterItems) {
+ if (TStringUtils.isBlank(filterItem)) {
+ continue;
+ }
+ filterSet.add(filterItem.trim());
+ }
+ }
+ topicAndFiltersMap.put(topicFilters[0].trim(), filterSet);
+ }
+ return topicAndFiltersMap;
+ }
+
+ public static byte[] buildTestData(int bodySize) {
+ final byte[] transmitData =
+ StringUtils.getBytesUtf8("This is a test data!");
+ final ByteBuffer dataBuffer = ByteBuffer.allocate(bodySize);
+ while (dataBuffer.hasRemaining()) {
+ int offset = dataBuffer.arrayOffset();
+ dataBuffer.put(transmitData, offset,
+ Math.min(dataBuffer.remaining(), transmitData.length));
+ }
+ dataBuffer.flip();
+ return dataBuffer.array();
+ }
}
diff --git a/tubemq-example/pom.xml b/tubemq-example/pom.xml
index 319acab..9ab6775 100644
--- a/tubemq-example/pom.xml
+++ b/tubemq-example/pom.xml
@@ -65,16 +65,6 @@
<groupId>org.apache.tubemq</groupId>
<artifactId>tubemq-client</artifactId>
</dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>${junit.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
</project>
diff --git a/tubemq-example/src/main/assembly/assembly.xml b/tubemq-example/src/main/assembly/assembly.xml
index 596af24..11edd32 100644
--- a/tubemq-example/src/main/assembly/assembly.xml
+++ b/tubemq-example/src/main/assembly/assembly.xml
@@ -32,9 +32,6 @@
<directory>../</directory>
<includes>
<include>./conf/tools.log4j.properties</include>
- <include>./bin/tubemq-consumer-perf-test.sh</include>
- <include>./bin/tubemq-producer-perf-test.sh</include>
- <include>./bin/env.sh</include>
<include>LICENSE</include>
<include>NOTICE</include>
<include>DISCLAIMER-WIP</include>
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/ArgsParserHelper.java b/tubemq-example/src/main/java/org/apache/tubemq/example/ArgsParserHelper.java
deleted file mode 100644
index c507ae4..0000000
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/ArgsParserHelper.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tubemq.example;
-
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-
-public class ArgsParserHelper {
-
- /**
- * Print help information and exit.
- *
- * @param opts - options
- */
- public static void help(String commandName, Options opts) {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(commandName, opts);
- System.exit(0);
- }
-
- /**
- * Init common options when parsing args.
- * @return - options
- */
- public static Options initCommonOptions() {
- Options options = new Options();
- options.addOption(null, "help", false, "show help");
- options.addOption(null, "master-list", true, "master address like: host1:8000,host2:8000");
- options.addOption(null, "topic", true, "topic list, topic1,topic2 or "
- + "topic1:tid11;tid12,topic2:tid21;tid22(consumer only)");
- return options;
- }
-}
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
index 9c76ce1..f76b077 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MAMessageProducerExample.java
@@ -30,13 +30,8 @@ import java.util.TreeSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.tubemq.client.config.TubeClientConfig;
import org.apache.tubemq.client.exception.TubeClientException;
@@ -64,7 +59,7 @@ public class MAMessageProducerExample {
private static final int SESSION_FACTORY_NUM = 10;
private static Set<String> topicSet;
- private static int batchCount;
+ private static int msgCount;
private static int producerCount;
private static byte[] sendData;
@@ -94,72 +89,45 @@ public class MAMessageProducerExample {
}
}
- /**
- * Init options
- *
- * @return options
- */
- public static Options initOptions() {
- Options options = ArgsParserHelper.initCommonOptions();
- options.addOption(null, "batch-size", true, "number of messages in single batch, default is 100000");
- options.addOption(null, "max-batch", true, "max batch number, default is 1024");
- options.addOption(null, "thread-num", true, "thread number of producers, default is 1, max is 100");
- return options;
- }
-
public static void main(String[] args) {
- Options options = null;
- try {
- CommandLineParser parser = new DefaultParser();
- options = initOptions();
- CommandLine cl = parser.parse(options, args);
- if (cl != null) {
- final String masterHostAndPort = cl.getOptionValue("master-list");
- final String topics = cl.getOptionValue("topic");
- final List<String> topicList = Arrays.asList(topics.split(","));
- topicSet = new TreeSet<>(topicList);
-
- batchCount = Integer.parseInt(cl.getOptionValue("max-batch", "100000"));
- int batchSize = Integer.parseInt(cl.getOptionValue("batch-size", "1024"));
- producerCount = Math.min(Integer.parseInt(cl.getOptionValue(
- "thread-num", "1")), MAX_PRODUCER_NUM);
- logger.info("MAMessageProducerExample.main started...");
- final byte[] transmitData = StringUtils
- .getBytesUtf8("This is a test message from multi-session factory.");
- final ByteBuffer dataBuffer = ByteBuffer.allocate(batchSize);
-
- while (dataBuffer.hasRemaining()) {
- int offset = dataBuffer.arrayOffset();
- dataBuffer.put(transmitData, offset,
- Math.min(dataBuffer.remaining(), transmitData.length));
- }
+ final String masterHostAndPort = args[0];
- dataBuffer.flip();
- sendData = dataBuffer.array();
+ final String topics = args[1];
+ final List<String> topicList = Arrays.asList(topics.split(","));
- try {
- MAMessageProducerExample messageProducer = new MAMessageProducerExample(
- masterHostAndPort);
+ topicSet = new TreeSet<>(topicList);
- messageProducer.startService();
+ msgCount = Integer.parseInt(args[2]);
+ producerCount = Math.min(args.length > 4 ? Integer.parseInt(args[3]) : 10, MAX_PRODUCER_NUM);
- while (SENT_SUCC_COUNTER.get() < (long) batchCount * producerCount * topicSet.size()) {
- TimeUnit.MILLISECONDS.sleep(1000);
- }
- messageProducer.producerMap.clear();
- messageProducer.shutdown();
+ logger.info("MAMessageProducerExample.main started...");
- } catch (TubeClientException e) {
- logger.error("TubeClientException: ", e);
- } catch (Throwable e) {
- logger.error("Throwable: ", e);
- }
- }
- } catch (Exception ex) {
- logger.error(ex.getMessage());
- if (options != null) {
- ArgsParserHelper.help("./tubemq-producer-perf-test.sh", options);
+ final byte[] transmitData = StringUtils.getBytesUtf8("This is a test message from multi-session factory.");
+ final ByteBuffer dataBuffer = ByteBuffer.allocate(1024);
+
+ while (dataBuffer.hasRemaining()) {
+ int offset = dataBuffer.arrayOffset();
+ dataBuffer.put(transmitData, offset, Math.min(dataBuffer.remaining(), transmitData.length));
+ }
+
+ dataBuffer.flip();
+ sendData = dataBuffer.array();
+
+ try {
+ MAMessageProducerExample messageProducer = new MAMessageProducerExample(masterHostAndPort);
+
+ messageProducer.startService();
+
+ while (SENT_SUCC_COUNTER.get() < msgCount * producerCount * topicSet.size()) {
+ Thread.sleep(1000);
}
+ messageProducer.producerMap.clear();
+ messageProducer.shutdown();
+
+ } catch (TubeClientException e) {
+ logger.error("TubeClientException: ", e);
+ } catch (Throwable e) {
+ logger.error("Throwable: ", e);
}
}
@@ -205,7 +173,7 @@ public class MAMessageProducerExample {
} catch (Throwable t) {
logger.error("publish exception: ", t);
}
- for (int i = 0; i < batchCount; i++) {
+ for (int i = 0; i < msgCount; i++) {
long millis = System.currentTimeMillis();
for (String topic : topicSet) {
try {
diff --git a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
index 3b99943..d9aeb8a 100644
--- a/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
+++ b/tubemq-example/src/main/java/org/apache/tubemq/example/MessageConsumerExample.java
@@ -26,10 +26,6 @@ import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
import org.apache.tubemq.client.common.PeerInfo;
import org.apache.tubemq.client.config.ConsumerConfig;
import org.apache.tubemq.client.consumer.ConsumePosition;
@@ -63,45 +59,29 @@ public final class MessageConsumerExample {
private static final MsgRecvStats msgRecvStats = new MsgRecvStats();
private final PushMessageConsumer messageConsumer;
+ private final MessageSessionFactory messageSessionFactory;
- public MessageConsumerExample(String masterHostAndPort, String group,
- int fetchCount, boolean isFromBegin) throws Exception {
+ public MessageConsumerExample(String masterHostAndPort, String group, int fetchCount) throws Exception {
ConsumerConfig consumerConfig = new ConsumerConfig(masterHostAndPort, group);
- if (isFromBegin) {
- consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_FIRST_OFFSET);
- } else {
- consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_LATEST_OFFSET);
- }
+ consumerConfig.setConsumePosition(ConsumePosition.CONSUMER_FROM_LATEST_OFFSET);
if (fetchCount > 0) {
consumerConfig.setPushFetchThreadCnt(fetchCount);
}
- MessageSessionFactory messageSessionFactory = new TubeSingleSessionFactory(consumerConfig);
+ this.messageSessionFactory = new TubeSingleSessionFactory(consumerConfig);
this.messageConsumer = messageSessionFactory.createPushConsumer(consumerConfig);
}
- /**
- * Init options
- * @return options
- */
- public static Options initOptions() {
-
- Options options = ArgsParserHelper.initCommonOptions();
- options.addOption(null, "batch-size", true, "max number of fetching message in one batch");
- options.addOption(null, "thread-num", true, "thread number of consumers");
- options.addOption(null, "group", true, "consumer group");
- options.addOption(null, "from-begin", false, "default is consuming from latest, "
- + "if option is clarified, then consume from begin");
- return options;
-
- }
+ public static void main(String[] args) {
+ final String masterHostAndPort = args[0];
+ final String topics = args[1];
+ final String group = args[2];
+ final int consumerCount = Integer.parseInt(args[3]);
+ int fetchCount = -1;
+ if (args.length > 5) {
+ fetchCount = Integer.parseInt(args[4]);
+ }
+ final Map<String, TreeSet<String>> topicTidsMap = new HashMap<>();
- /**
- * init topic->set(tid) map
- * @param topics - topics string
- * @return - map of topic->set(tid)
- */
- private static Map<String, TreeSet<String>> initTopicList(String topics) {
- Map<String, TreeSet<String>> topicTidsMap = new HashMap<>();
String[] topicTidsList = topics.split(",");
for (String topicTids : topicTidsList) {
String[] topicTidStr = topicTids.split(":");
@@ -115,60 +95,35 @@ public final class MessageConsumerExample {
}
topicTidsMap.put(topicTidStr[0], tids);
}
- return topicTidsMap;
- }
+ final int startFetchCount = fetchCount;
+ final ExecutorService executorService = Executors.newFixedThreadPool(fetchCount);
+ for (int i = 0; i < consumerCount; i++) {
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ MessageConsumerExample messageConsumer = new MessageConsumerExample(
+ masterHostAndPort,
+ group,
+ startFetchCount
+ );
+ messageConsumer.subscribe(topicTidsMap);
+ } catch (Exception e) {
+ logger.error("Create consumer failed!", e);
+ }
+ }
+ });
+ }
+ final Thread statisticThread = new Thread(msgRecvStats, "Received Statistic Thread");
+ statisticThread.start();
- public static void main(String[] args) {
- Options options = null;
+ executorService.shutdown();
try {
- CommandLineParser parser = new DefaultParser();
- options = initOptions();
- CommandLine cl = parser.parse(options, args);
- if (cl != null) {
- final String masterHostAndPort = cl.getOptionValue("master-list");
- final Map<String, TreeSet<String>> topicTidsMap = initTopicList(
- cl.getOptionValue("topic"));
- final String group = cl.getOptionValue("group");
- int threadNum = Integer.parseInt(cl.getOptionValue("thread-num", "1"));
- final int fetchCount = Integer.parseInt(cl.getOptionValue("batch-size", "-1"));
- final boolean isFromBegin = cl.hasOption("from-begin");
- ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
- for (int i = 0; i < threadNum; i++) {
- executorService.submit(new Runnable() {
- @Override
- public void run() {
- try {
- MessageConsumerExample messageConsumer = new MessageConsumerExample(
- masterHostAndPort,
- group,
- fetchCount,
- isFromBegin
- );
- messageConsumer.subscribe(topicTidsMap);
- } catch (Exception e) {
- logger.error("Create consumer failed!", e);
- }
- }
- });
- }
- final Thread statisticThread = new Thread(msgRecvStats,
- "Received Statistic Thread");
- statisticThread.start();
-
- executorService.shutdown();
- try {
- executorService.awaitTermination(60 * 1000, TimeUnit.MILLISECONDS);
- } catch (InterruptedException e) {
- logger.error("Thread Pool shutdown has been interrupted!");
- }
- msgRecvStats.stopStats();
- }
- } catch (Exception ex) {
- logger.error(ex.getMessage(), ex.getMessage());
- if (options != null) {
- ArgsParserHelper.help("./tubemq-consumer-perf-test.sh", options);
- }
+ executorService.awaitTermination(60 * 1000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ logger.error("Thread Pool shutdown has been interrupted!");
}
+ msgRecvStats.stopStats();
}
public void subscribe(Map<String, TreeSet<String>> topicTidsMap) throws TubeClientException {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
index b2d9327..abb2e2a 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
@@ -27,64 +27,76 @@ public enum CliArgDef {
HELP("h", "help", "Print usage information."),
VERSION("v", "version", "Display TubeMQ version."),
- MASTERSERVER("master-servers", "master-servers",
+ MASTERSERVER(null, "master-servers",
"String: format is master1_ip:port[,master2_ip:port]",
"The master address(es) to connect to."),
- MASTERURL("master-url", "master-url",
+ MASTERURL(null, "master-url",
"String: format is http://master_ip:master_webport/",
"Master Service URL to which to connect.(default: http://localhost:8080/)"),
- BROKERURL("broker-url", "broker-url",
+ BROKERURL(null, "broker-url",
"String: format is http://broker_ip:broker_webport/",
"Broker Service URL to which to connect.(default: http://localhost:8081/)"),
- MESSAGES("messages", "messages",
+ MESSAGES(null, "messages",
"Long: count",
"The number of messages to send or consume, If not set, production or consumption is continual."),
- MSGDATASIZE("msg-data-size", "message-data-size",
- "Int: message size",
+ MSGDATASIZE(null, "message-data-size",
+ "Int: message size,(0, 1024 * 1024)",
"message's data size in bytes. Note that you must provide exactly"
+ " one of --msg-data-size or --payload-file."),
- PAYLOADFILE("payload-file", "payload-file",
+ PAYLOADFILE(null, "payload-file",
"String: payload file path",
"file to read the message payloads from. This works only for"
+ " UTF-8 encoded text files. Payloads will be read from this"
+ " file and a payload will be randomly selected when sending"
+ " messages. Note that you must provide exactly one"
+ " of --msg-data-size or --payload-file."),
- PAYLOADDELIM("payload-delimiter", "payload-delimiter",
+ PAYLOADDELIM(null, "payload-delimiter",
"String: payload data's delimiter",
"provides delimiter to be used when --payload-file is provided."
+ " Defaults to new line. Note that this parameter will be"
+ " ignored if --payload-file is not provided. (default: \\n)"),
PRDTOPIC("topic", "topicName",
- "String: topic, format is topic_1[,topic_2[:filterCond_2.1[;filterCond_2.2]]]",
+ "String: topic, format is topic_1[,topic_2[:filterCond_2.1[\\;filterCond_2.2]]]",
"The topic(s) to produce messages to."),
CNSTOPIC("topic", "topicName",
- "String: topic, format is topic_1[[:filterCond_1.1[;filterCond_1.2]][,topic_2]]",
+ "String: topic, format is topic_1[[:filterCond_1.1[\\;filterCond_1.2]][,topic_2]]",
"The topic(s) to consume on."),
- RPCTIMEOUT("timeout", "timeout",
+ RPCTIMEOUT(null, "rpc-timeout",
"Long: milliseconds",
"The maximum duration between request and response in milliseconds. (default: 10000)"),
+ CONNREUSE(null, "conn-reuse",
+ "bool: true or false",
+ "Different clients reuse TCP connections. (default: true)"),
GROUP("group", "groupName",
"String: consumer group",
- "The consumer group name of the consumer."),
- CLIENTCOUNT("client-num", "client-num",
- "Int: client count",
- "Number of consumers to started."),
- PULLMODEL("pull-model", "pull-model",
- "Pull consumption model."),
- PUSHMODEL("push-model", "push-model",
- "Push consumption model."),
- FETCHTHREADS("num-fetch-threads", "num-fetch-threads",
- "Integer: count",
+ "The consumer group name of the consumer. (default: test_consume)"),
+ CLIENTCOUNT(null, "client-count",
+ "Int: client count, [1, 100]",
+ "Number of producers or consumers to started."),
+ PUSHCONSUME(null, "consume-push",
+ "Push consumption action.(default: pull)"),
+ FETCHTHREADS(null, "num-fetch-threads",
+ "Integer: count, [1,100]",
"Number of fetch threads, default: num of cpu count."),
- FROMLATEST("from-latest", "from-latest",
- "Start to consume from the latest message present in the log."),
- FROMBEGINNING("from-beginning", "from-beginning",
- "If the consumer does not already have an established offset to consume from,"
- + " start with the earliest message present in the log rather than the latest message."),
- OUTPUTINTERVAL("output-interval", "output-interval",
- "Integer: interval_ms",
- "Interval in milliseconds at which to print progress info. (default: 5000)");
+ SENDTHREADS(null, "num-send-threads",
+ "Integer: count, [1,200]",
+ "Number of send message threads, default: num of cpu count."),
+ CONSUMEPOS(null, "consume-position",
+ "Integer: [-1,0, 1]",
+ "Set the start position of the consumer group. The value can be [-1, 0, 1]."
+ + " Default value is 0. -1: Start from 0 for the first time."
+ + " Otherwise start from last consume position."
+ + " 0: Start from the latest position for the first time."
+ + " Otherwise start from last consume position."
+ + " 1: Start from the latest consume position."),
+ OUTPUTINTERVAL(null, "output-interval",
+ "Integer: interval_ms, [5000, +)",
+ "Interval in milliseconds at which to print progress info. (default: 5000)"),
+ SYNCPRODUCE(null, "sync-produce",
+ "Synchronous production. (default: false)"),
+ WITHOUTDELAY(null, "without-delay",
+ "Production without delay. (default: false)");
+
CliArgDef(String opt, String longOpt, String optDesc) {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java
new file mode 100644
index 0000000..c0903f2
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.tools.cli;
+
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.tubemq.server.common.TubeServerVersion;
+import org.apache.tubemq.server.common.fielddef.CliArgDef;
+
+
+public abstract class CliAbstractBase {
+
+ protected final String commandName;
+ protected Options options = new Options();
+ protected CommandLineParser parser = new DefaultParser();
+ private HelpFormatter formatter = new HelpFormatter();
+
+ public CliAbstractBase(String commandName) {
+ this.commandName = commandName;
+ addCommandOption(CliArgDef.HELP);
+ addCommandOption(CliArgDef.VERSION);
+ formatter.setWidth(500);
+ }
+
+ /**
+ * Print help information and exit.
+ *
+ */
+ public void help() {
+ formatter.printHelp(commandName, options);
+ System.exit(0);
+ }
+
+ /**
+ * Print tubemq server version.
+ *
+ */
+ public void version() {
+ System.out.println("TubeMQ " + TubeServerVersion.BROKER_VERSION);
+ System.exit(0);
+ }
+
+ public void addCommandOption(CliArgDef cliArgDef) {
+ Option option = new Option(cliArgDef.opt,
+ cliArgDef.longOpt, cliArgDef.hasArg, cliArgDef.optDesc);
+ if (cliArgDef.hasArg) {
+ option.setArgName(cliArgDef.argDesc);
+ }
+ options.addOption(option);
+ }
+
+
+ protected abstract void initCommandOptions();
+
+
+ public abstract boolean parseParams(String[] args) throws Exception;
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
new file mode 100644
index 0000000..666e6f9
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
@@ -0,0 +1,394 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.tools.cli;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
+import org.apache.tubemq.client.common.PeerInfo;
+import org.apache.tubemq.client.config.ConsumerConfig;
+import org.apache.tubemq.client.consumer.ConsumePosition;
+import org.apache.tubemq.client.consumer.ConsumerResult;
+import org.apache.tubemq.client.consumer.MessageConsumer;
+import org.apache.tubemq.client.consumer.MessageV2Listener;
+import org.apache.tubemq.client.consumer.PullMessageConsumer;
+import org.apache.tubemq.client.consumer.PushMessageConsumer;
+import org.apache.tubemq.client.exception.TubeClientException;
+import org.apache.tubemq.client.factory.MessageSessionFactory;
+import org.apache.tubemq.client.factory.TubeMultiSessionFactory;
+import org.apache.tubemq.client.factory.TubeSingleSessionFactory;
+import org.apache.tubemq.corebase.Message;
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.TErrCodeConstants;
+import org.apache.tubemq.corebase.utils.MixedUtils;
+import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.ThreadUtils;
+import org.apache.tubemq.server.common.fielddef.CliArgDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+/**
+ * This class is use to process CLI Consumer process.
+ *
+ *
+ */
+public class CliConsumer extends CliAbstractBase {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(CliConsumer.class);
+ // statistic data index
+ private static final AtomicLong TOTAL_COUNTER = new AtomicLong(0);
+ // sent data content
+ private final Map<String, TreeSet<String>> topicAndFiltersMap = new HashMap<>();
+ private final List<MessageSessionFactory> sessionFactoryList = new ArrayList<>();
+ private final Map<MessageConsumer, TupleValue> consumerMap = new HashMap<>();
+ // cli parameters
+ private String masterServers;
+ private String groupName = "test_consume";
+ private ConsumePosition consumePos =
+ ConsumePosition.CONSUMER_FROM_LATEST_OFFSET;
+ private long msgCount = TBaseConstants.META_VALUE_UNDEFINED;
+ private long rpcTimeoutMs = TBaseConstants.META_VALUE_UNDEFINED;
+ private boolean reuseConn = false;
+ private int clientCount = 1;
+ private int fetchThreadCnt =
+ Runtime.getRuntime().availableProcessors();
+ private long printIntervalMs = 5000;
+ private boolean isPushConsume = false;
+
+ private boolean isStarted = false;
+
+
+ public CliConsumer() {
+ super("tubemq-consumer-test.sh");
+ initCommandOptions();
+ }
+
+ /**
+ * Init command options
+ */
+ protected void initCommandOptions() {
+ // add the cli required parameters
+ addCommandOption(CliArgDef.MASTERSERVER);
+ addCommandOption(CliArgDef.MESSAGES);
+ addCommandOption(CliArgDef.CNSTOPIC);
+ addCommandOption(CliArgDef.RPCTIMEOUT);
+ addCommandOption(CliArgDef.GROUP);
+ addCommandOption(CliArgDef.CONNREUSE);
+ addCommandOption(CliArgDef.PUSHCONSUME);
+ addCommandOption(CliArgDef.CONSUMEPOS);
+ addCommandOption(CliArgDef.FETCHTHREADS);
+ addCommandOption(CliArgDef.CLIENTCOUNT);
+ addCommandOption(CliArgDef.OUTPUTINTERVAL);
+ addCommandOption(CliArgDef.WITHOUTDELAY);
+ }
+
+ public boolean parseParams(String[] args) throws Exception {
+ // parse parameters and check value
+ CommandLine cli = parser.parse(options, args);
+ if (cli == null) {
+ throw new ParseException("Parse args failure");
+ }
+ if (cli.hasOption(CliArgDef.VERSION.longOpt)) {
+ version();
+ }
+ if (cli.hasOption(CliArgDef.HELP.longOpt)) {
+ help();
+ }
+ masterServers = cli.getOptionValue(CliArgDef.MASTERSERVER.longOpt);
+ if (TStringUtils.isBlank(masterServers)) {
+ throw new Exception(CliArgDef.MASTERSERVER.longOpt + " is required!");
+ }
+ String topicStr = cli.getOptionValue(CliArgDef.CNSTOPIC.longOpt);
+ if (TStringUtils.isBlank(topicStr)) {
+ throw new Exception(CliArgDef.CNSTOPIC.longOpt + " is required!");
+ }
+ topicAndFiltersMap.putAll(MixedUtils.parseTopicParam(topicStr));
+ if (topicAndFiltersMap.isEmpty()) {
+ throw new Exception("Invalid " + CliArgDef.CNSTOPIC.longOpt + " parameter value!");
+ }
+ String msgCntStr = cli.getOptionValue(CliArgDef.MESSAGES.longOpt);
+ if (TStringUtils.isNotBlank(msgCntStr)) {
+ msgCount = Long.parseLong(msgCntStr);
+ }
+ String groupNameStr = cli.getOptionValue(CliArgDef.GROUP.longOpt);
+ if (TStringUtils.isNotBlank(groupNameStr)) {
+ groupName = cli.getOptionValue(CliArgDef.GROUP.longOpt);
+ }
+ String reuseConnStr = cli.getOptionValue(CliArgDef.CONNREUSE.longOpt);
+ if (TStringUtils.isNotBlank(reuseConnStr)) {
+ reuseConn = Boolean.parseBoolean(reuseConnStr);
+ }
+ String rpcTimeoutStr = cli.getOptionValue(CliArgDef.RPCTIMEOUT.longOpt);
+ if (TStringUtils.isNotBlank(rpcTimeoutStr)) {
+ rpcTimeoutMs = Long.parseLong(rpcTimeoutStr);
+ }
+ String clientCntStr = cli.getOptionValue(CliArgDef.CLIENTCOUNT.longOpt);
+ if (TStringUtils.isNotBlank(clientCntStr)) {
+ clientCount = Integer.parseInt(clientCntStr);
+ }
+ String printIntMsStr = cli.getOptionValue(CliArgDef.OUTPUTINTERVAL.longOpt);
+ if (TStringUtils.isNotBlank(printIntMsStr)) {
+ printIntervalMs = Long.parseLong(printIntMsStr);
+ if (printIntervalMs < 5000) {
+ throw new Exception("Invalid "
+ + CliArgDef.OUTPUTINTERVAL.longOpt
+ + " parameter value!");
+ }
+ }
+ String consumePosStr = cli.getOptionValue(CliArgDef.CONSUMEPOS.longOpt);
+ if (TStringUtils.isNotBlank(consumePosStr)) {
+ int tmpPosId = Integer.parseInt(consumePosStr);
+ if (tmpPosId > 0) {
+ consumePos = ConsumePosition.CONSUMER_FROM_MAX_OFFSET_ALWAYS;
+ } else if (tmpPosId < 0) {
+ consumePos = ConsumePosition.CONSUMER_FROM_FIRST_OFFSET;
+ } else {
+ consumePos = ConsumePosition.CONSUMER_FROM_LATEST_OFFSET;
+ }
+ }
+ if (cli.hasOption(CliArgDef.PUSHCONSUME.longOpt)) {
+ isPushConsume = true;
+ }
+ String fetchThreadCntStr = cli.getOptionValue(CliArgDef.FETCHTHREADS.longOpt);
+ if (TStringUtils.isNotBlank(fetchThreadCntStr)) {
+ int tmpFetchThreadCnt = Integer.parseInt(fetchThreadCntStr);
+ tmpFetchThreadCnt = (tmpFetchThreadCnt < 1) ? 1 : Math.min(tmpFetchThreadCnt, 100);
+ fetchThreadCnt = tmpFetchThreadCnt;
+ }
+ return true;
+ }
+
+ // initial tubemq client order by caller required
+ public void initTask() throws Exception {
+ // initial consumer configure
+ ConsumerConfig consumerConfig =
+ new ConsumerConfig(masterServers, groupName);
+ consumerConfig.setRpcTimeoutMs(rpcTimeoutMs);
+ consumerConfig.setPushFetchThreadCnt(fetchThreadCnt);
+ consumerConfig.setConsumePosition(consumePos);
+ // initial consumer object
+ if (isPushConsume) {
+ DefaultMessageListener msgListener =
+ new DefaultMessageListener();
+ if (reuseConn) {
+ // if reuse connection, need use TubeSingleSessionFactory class
+ MessageSessionFactory msgSessionFactory =
+ new TubeSingleSessionFactory(consumerConfig);
+ this.sessionFactoryList.add(msgSessionFactory);
+ for (int i = 0; i < clientCount; i++) {
+ PushMessageConsumer consumer1 =
+ msgSessionFactory.createPushConsumer(consumerConfig);
+ for (Map.Entry<String, TreeSet<String>> entry
+ : topicAndFiltersMap.entrySet()) {
+ consumer1.subscribe(entry.getKey(), entry.getValue(), msgListener);
+ }
+ consumer1.completeSubscribe();
+ consumerMap.put(consumer1, null);
+ }
+ } else {
+ for (int i = 0; i < clientCount; i++) {
+ MessageSessionFactory msgSessionFactory =
+ new TubeMultiSessionFactory(consumerConfig);
+ this.sessionFactoryList.add(msgSessionFactory);
+ PushMessageConsumer consumer1 =
+ msgSessionFactory.createPushConsumer(consumerConfig);
+ for (Map.Entry<String, TreeSet<String>> entry
+ : topicAndFiltersMap.entrySet()) {
+ consumer1.subscribe(entry.getKey(), entry.getValue(), msgListener);
+ }
+ consumer1.completeSubscribe();
+ consumerMap.put(consumer1, null);
+ }
+ }
+ } else {
+ if (reuseConn) {
+ MessageSessionFactory msgSessionFactory =
+ new TubeSingleSessionFactory(consumerConfig);
+ this.sessionFactoryList.add(msgSessionFactory);
+ for (int i = 0; i < clientCount; i++) {
+ PullMessageConsumer consumer2 =
+ msgSessionFactory.createPullConsumer(consumerConfig);
+ for (Map.Entry<String, TreeSet<String>> entry
+ : topicAndFiltersMap.entrySet()) {
+ consumer2.subscribe(entry.getKey(), entry.getValue());
+ }
+ consumer2.completeSubscribe();
+ consumerMap.put(consumer2,
+ new TupleValue(consumer2, msgCount, fetchThreadCnt));
+ }
+ } else {
+ for (int i = 0; i < clientCount; i++) {
+ MessageSessionFactory msgSessionFactory =
+ new TubeMultiSessionFactory(consumerConfig);
+ this.sessionFactoryList.add(msgSessionFactory);
+ PullMessageConsumer consumer2 =
+ msgSessionFactory.createPullConsumer(consumerConfig);
+ for (Map.Entry<String, TreeSet<String>> entry
+ : topicAndFiltersMap.entrySet()) {
+ consumer2.subscribe(entry.getKey(), entry.getValue());
+ }
+ consumer2.completeSubscribe();
+ consumerMap.put(consumer2,
+ new TupleValue(consumer2, msgCount, fetchThreadCnt));
+ }
+ }
+ }
+ isStarted = true;
+ }
+
+ public void shutdown() throws Throwable {
+ // stop process
+ ThreadUtils.sleep(20);
+ for (MessageConsumer consumer : consumerMap.keySet()) {
+ consumer.shutdown();
+ }
+ for (MessageSessionFactory messageSessionFactory : sessionFactoryList) {
+ messageSessionFactory.shutdown();
+ }
+ }
+
+
+ private static class TupleValue {
+ public Thread[] fetchRunners = null;
+
+ public TupleValue(PullMessageConsumer consumer, long msgCount, int fetchThreadCnt) {
+ fetchRunners = new Thread[fetchThreadCnt];
+ for (int i = 0; i < fetchRunners.length; i++) {
+ fetchRunners[i] = new Thread(new FetchRequestRunner(consumer, msgCount));
+ fetchRunners[i].setName("_fetch_runner_" + i);
+ }
+ for (Thread thread : fetchRunners) {
+ thread.start();
+ }
+ }
+
+ }
+
+
+ // for push consumer callback process
+ private static class DefaultMessageListener implements MessageV2Listener {
+
+ public DefaultMessageListener() {
+ }
+
+ @Override
+ public void receiveMessages(PeerInfo peerInfo, List<Message> messages) {
+ if (messages != null && !messages.isEmpty()) {
+ TOTAL_COUNTER.addAndGet(messages.size());
+ }
+ }
+
+ @Override
+ public void receiveMessages(List<Message> messages) {
+ // deprecated
+ }
+
+ @Override
+ public Executor getExecutor() {
+ return null;
+ }
+
+ @Override
+ public void stop() {
+ }
+ }
+
+ // for push consumer process
+ private static class FetchRequestRunner implements Runnable {
+
+ private final PullMessageConsumer messageConsumer;
+ private final long msgConsumeCnt;
+
+ FetchRequestRunner(PullMessageConsumer messageConsumer, long msgConsumeCnt) {
+ this.messageConsumer = messageConsumer;
+ this.msgConsumeCnt = msgConsumeCnt;
+ }
+
+ @Override
+ public void run() {
+ try {
+ do {
+ ConsumerResult result = messageConsumer.getMessage();
+ if (result.isSuccess()) {
+ List<Message> messageList = result.getMessageList();
+ if (messageList != null && !messageList.isEmpty()) {
+ TOTAL_COUNTER.addAndGet(messageList.size());
+ }
+ messageConsumer.confirmConsume(result.getConfirmContext(), true);
+ } else {
+ if (!TErrCodeConstants.IGNORE_ERROR_SET.contains(result.getErrCode())) {
+ logger.info(
+ "Receive messages errorCode is {}, Error message is {}",
+ result.getErrCode(),
+ result.getErrMsg());
+ if (messageConsumer.isShutdown()) {
+ break;
+ }
+ }
+ }
+ if (msgConsumeCnt >= 0) {
+ if (TOTAL_COUNTER.get() >= msgConsumeCnt) {
+ break;
+ }
+ }
+ } while (true);
+ } catch (TubeClientException e) {
+ logger.error("Create consumer failed!", e);
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ CliConsumer cliConsumer = new CliConsumer();
+ try {
+ boolean result = cliConsumer.parseParams(args);
+ if (!result) {
+ throw new Exception("Parse parameters failure!");
+ }
+ cliConsumer.initTask();
+ ThreadUtils.sleep(1000);
+ while (cliConsumer.msgCount < 0
+ || TOTAL_COUNTER.get() < cliConsumer.msgCount * cliConsumer.clientCount) {
+ ThreadUtils.sleep(cliConsumer.printIntervalMs);
+ System.out.println("Required received count VS received message count = "
+ + (cliConsumer.msgCount * cliConsumer.clientCount)
+ + " : " + TOTAL_COUNTER.get());
+ }
+ cliConsumer.shutdown();
+ System.out.println("Finished, received count VS received message count = "
+ + (cliConsumer.msgCount * cliConsumer.clientCount)
+ + " : " + TOTAL_COUNTER.get());
+ } catch (Throwable ex) {
+ ex.printStackTrace();
+ logger.error(ex.getMessage());
+ cliConsumer.help();
+ }
+
+ }
+
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
new file mode 100644
index 0000000..2734ac1
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
@@ -0,0 +1,385 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.tools.cli;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
+import org.apache.tubemq.client.config.TubeClientConfig;
+import org.apache.tubemq.client.factory.MessageSessionFactory;
+import org.apache.tubemq.client.factory.TubeMultiSessionFactory;
+import org.apache.tubemq.client.factory.TubeSingleSessionFactory;
+import org.apache.tubemq.client.producer.MessageProducer;
+import org.apache.tubemq.client.producer.MessageSentCallback;
+import org.apache.tubemq.client.producer.MessageSentResult;
+import org.apache.tubemq.corebase.Message;
+import org.apache.tubemq.corebase.TBaseConstants;
+import org.apache.tubemq.corebase.utils.MixedUtils;
+import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.corebase.utils.ThreadUtils;
+import org.apache.tubemq.server.common.fielddef.CliArgDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * This class is use to process CLI Producer process.
+ *
+ *
+ */
+public class CliProducer extends CliAbstractBase {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(CliProducer.class);
+ // statistic data index
+ private static final AtomicLong TOTAL_COUNTER = new AtomicLong(0);
+ private static final AtomicLong SENT_SUCC_COUNTER = new AtomicLong(0);
+ private static final AtomicLong SENT_FAIL_COUNTER = new AtomicLong(0);
+ private static final AtomicLong SENT_EXCEPT_COUNTER = new AtomicLong(0);
+ // sent data content
+ private static byte[] sentData;
+ private final Map<String, TreeSet<String>> topicAndFiltersMap = new HashMap<>();
+ private final List<TupleValue> topicSendRounds = new ArrayList<>();
+ private final List<MessageSessionFactory> sessionFactoryList = new ArrayList<>();
+ private final Map<MessageProducer, MsgSender> producerMap = new HashMap<>();
+ // cli parameters
+ private String masterServers;
+ private long msgCount = TBaseConstants.META_VALUE_UNDEFINED;
+ private boolean useRandData = true;
+ private int msgDataSize = 1000;
+ private String payloadFilePath = null;
+ private String payloadDelim = null;
+ private long rpcTimeoutMs = TBaseConstants.META_VALUE_UNDEFINED;
+ private boolean reuseConn = false;
+ private int clientCount = 1;
+ private int sendThreadCnt = 100;
+ private long printIntervalMs = 5000;
+ private boolean syncProduction = false;
+ private boolean withoutDelay = false;
+ private boolean isStarted = false;
+ private ExecutorService sendExecutorService = null;
+
+
+ public CliProducer() {
+ super("tubemq-producer-test.sh");
+ initCommandOptions();
+ }
+
+ /**
+ * Init command options
+ */
+ protected void initCommandOptions() {
+ // add the cli required parameters
+ addCommandOption(CliArgDef.MASTERSERVER);
+ addCommandOption(CliArgDef.MESSAGES);
+ addCommandOption(CliArgDef.MSGDATASIZE);
+ //addCommandOption(CliArgDef.PAYLOADFILE);
+ //addCommandOption(CliArgDef.PAYLOADDELIM);
+ addCommandOption(CliArgDef.PRDTOPIC);
+ addCommandOption(CliArgDef.RPCTIMEOUT);
+ addCommandOption(CliArgDef.CONNREUSE);
+ addCommandOption(CliArgDef.CLIENTCOUNT);
+ addCommandOption(CliArgDef.OUTPUTINTERVAL);
+ addCommandOption(CliArgDef.SYNCPRODUCE);
+ addCommandOption(CliArgDef.SENDTHREADS);
+ addCommandOption(CliArgDef.WITHOUTDELAY);
+ }
+
+ public boolean parseParams(String[] args) throws Exception {
+ // parse parameters and check value
+ CommandLine cli = parser.parse(options, args);
+ if (cli == null) {
+ throw new ParseException("Parse args failure");
+ }
+ if (cli.hasOption(CliArgDef.VERSION.longOpt)) {
+ version();
+ }
+ if (cli.hasOption(CliArgDef.HELP.longOpt)) {
+ help();
+ }
+ masterServers = cli.getOptionValue(CliArgDef.MASTERSERVER.longOpt);
+ if (TStringUtils.isBlank(masterServers)) {
+ throw new Exception(CliArgDef.MASTERSERVER.longOpt + " is required!");
+ }
+ String topicStr = cli.getOptionValue(CliArgDef.PRDTOPIC.longOpt);
+ if (TStringUtils.isBlank(topicStr)) {
+ throw new Exception(CliArgDef.PRDTOPIC.longOpt + " is required!");
+ }
+ topicAndFiltersMap.putAll(MixedUtils.parseTopicParam(topicStr));
+ if (topicAndFiltersMap.isEmpty()) {
+ throw new Exception("Invalid " + CliArgDef.PRDTOPIC.longOpt + " parameter value!");
+ }
+ String msgCntStr = cli.getOptionValue(CliArgDef.MESSAGES.longOpt);
+ if (TStringUtils.isNotBlank(msgCntStr)) {
+ msgCount = Long.parseLong(msgCntStr);
+ }
+ String msgDataSizeStr = cli.getOptionValue(CliArgDef.MSGDATASIZE.longOpt);
+ if (TStringUtils.isNotBlank(msgDataSizeStr)) {
+ msgDataSize = Integer.parseInt(msgDataSizeStr);
+ }
+ String reuseConnStr = cli.getOptionValue(CliArgDef.CONNREUSE.longOpt);
+ if (TStringUtils.isNotBlank(reuseConnStr)) {
+ reuseConn = Boolean.parseBoolean(reuseConnStr);
+ }
+ String sendThreadCntStr = cli.getOptionValue(CliArgDef.SENDTHREADS.longOpt);
+ if (TStringUtils.isNotBlank(sendThreadCntStr)) {
+ int tmpThreadCnt = Integer.parseInt(sendThreadCntStr);
+ tmpThreadCnt = (tmpThreadCnt < 1) ? 1 : Math.min(tmpThreadCnt, 200);
+ sendThreadCnt = tmpThreadCnt;
+ }
+ String rpcTimeoutStr = cli.getOptionValue(CliArgDef.RPCTIMEOUT.longOpt);
+ if (TStringUtils.isNotBlank(rpcTimeoutStr)) {
+ rpcTimeoutMs = Long.parseLong(rpcTimeoutStr);
+ }
+ String clientCntStr = cli.getOptionValue(CliArgDef.CLIENTCOUNT.longOpt);
+ if (TStringUtils.isNotBlank(clientCntStr)) {
+ clientCount = Integer.parseInt(clientCntStr);
+ }
+ String printIntMsStr = cli.getOptionValue(CliArgDef.OUTPUTINTERVAL.longOpt);
+ if (TStringUtils.isNotBlank(printIntMsStr)) {
+ printIntervalMs = Long.parseLong(printIntMsStr);
+ if (printIntervalMs < 5000) {
+ throw new Exception("Invalid "
+ + CliArgDef.OUTPUTINTERVAL.longOpt
+ + " parameter value!");
+ }
+ }
+ if (cli.hasOption(CliArgDef.SYNCPRODUCE.longOpt)) {
+ syncProduction = true;
+ }
+ if (cli.hasOption(CliArgDef.WITHOUTDELAY.longOpt)) {
+ withoutDelay = true;
+ }
+ return true;
+ }
+ // initial tubemq client order by caller required
+ public void initTask() throws Exception {
+ // initial sent data
+ sentData = MixedUtils.buildTestData(msgDataSize);
+ // initial client configure
+ TubeClientConfig clientConfig = new TubeClientConfig(masterServers);
+ clientConfig.setRpcTimeoutMs(rpcTimeoutMs);
+ // initial topic send round
+ for (Map.Entry<String, TreeSet<String>> entry: topicAndFiltersMap.entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ topicSendRounds.add(new TupleValue(entry.getKey()));
+ } else {
+ for (String filter : entry.getValue()) {
+ topicSendRounds.add(new TupleValue(entry.getKey(), filter));
+ }
+ }
+ }
+ // initial send thread service
+ sendExecutorService =
+ Executors.newFixedThreadPool(sendThreadCnt, new ThreadFactory() {
+ @Override
+ public Thread newThread(Runnable runnable) {
+ return new Thread(runnable, "sender_" + producerMap.size());
+ }
+ });
+ // initial producer object
+ if (reuseConn) {
+ // if resue connection, use TubeSingleSessionFactory class
+ MessageSessionFactory msgSessionFactory =
+ new TubeSingleSessionFactory(clientConfig);
+ this.sessionFactoryList.add(msgSessionFactory);
+ for (int i = 0; i < clientCount; i++) {
+ MessageProducer producer = msgSessionFactory.createProducer();
+ producer.publish(topicAndFiltersMap.keySet());
+ producerMap.put(producer, new MsgSender(producer));
+ // send send task
+ sendExecutorService.submit(producerMap.get(producer));
+ }
+ } else {
+ for (int i = 0; i < clientCount; i++) {
+ // if not resue connection, use TubeMultiSessionFactory class
+ MessageSessionFactory msgSessionFactory =
+ new TubeMultiSessionFactory(clientConfig);
+ this.sessionFactoryList.add(msgSessionFactory);
+ MessageProducer producer = msgSessionFactory.createProducer();
+ producer.publish(topicAndFiltersMap.keySet());
+ producerMap.put(producer, new MsgSender(producer));
+ // send send task
+ sendExecutorService.submit(producerMap.get(producer));
+ }
+ }
+ isStarted = true;
+ }
+
+ public void shutdown() throws Throwable {
+ // stop process
+ if (sendExecutorService != null) {
+ sendExecutorService.shutdownNow();
+ }
+ ThreadUtils.sleep(20);
+ for (MessageProducer producer : producerMap.keySet()) {
+ producer.shutdown();
+ }
+ for (MessageSessionFactory messageSessionFactory : sessionFactoryList) {
+ messageSessionFactory.shutdown();
+ }
+ }
+
+ // process message send
+ public class MsgSender implements Runnable {
+
+ private final MessageProducer producer;
+
+ public MsgSender(MessageProducer producer) {
+ this.producer = producer;
+ }
+
+ @Override
+ public void run() {
+ int topicAndCondCnt = topicSendRounds.size();
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmm");
+ long sentCount = 0;
+ int roundIndex = 0;
+ while (msgCount < 0 || sentCount < msgCount) {
+ roundIndex = (int) (sentCount++ % topicAndCondCnt);
+ try {
+ long millis = System.currentTimeMillis();
+ TupleValue tupleValue = topicSendRounds.get(roundIndex);
+ Message message = new Message(tupleValue.topic, sentData);
+ if (tupleValue.filter != null) {
+ // if include filter, add filter item
+ message.putSystemHeader(tupleValue.filter, sdf.format(new Date(millis)));
+ }
+ // use sync or async process
+ if (syncProduction) {
+ MessageSentResult procResult =
+ producer.sendMessage(message);
+ TOTAL_COUNTER.incrementAndGet();
+ if (procResult.isSuccess()) {
+ SENT_SUCC_COUNTER.incrementAndGet();
+ } else {
+ SENT_FAIL_COUNTER.incrementAndGet();
+ }
+ } else {
+ producer.sendMessage(message, new DefaultSendCallback());
+ }
+ } catch (Throwable e1) {
+ TOTAL_COUNTER.incrementAndGet();
+ SENT_EXCEPT_COUNTER.incrementAndGet();
+ logger.error("sendMessage exception: ", e1);
+ }
+ // Limit sending flow control to avoid frequent errors
+ // caused by too many inflight messages being sent
+ if (!withoutDelay) {
+ if (sentCount % 5000 == 0) {
+ ThreadUtils.sleep(3000);
+ } else if (sentCount % 4000 == 0) {
+ ThreadUtils.sleep(2000);
+ } else if (sentCount % 2000 == 0) {
+ ThreadUtils.sleep(800);
+ } else if (sentCount % 1000 == 0) {
+ ThreadUtils.sleep(400);
+ }
+ }
+ }
+ // finished, close client
+ try {
+ producer.shutdown();
+ } catch (Throwable e) {
+ logger.error("producer shutdown error: ", e);
+ }
+ }
+ }
+
+ private class DefaultSendCallback implements MessageSentCallback {
+ @Override
+ public void onMessageSent(MessageSentResult result) {
+ TOTAL_COUNTER.incrementAndGet();
+ if (result.isSuccess()) {
+ SENT_SUCC_COUNTER.incrementAndGet();
+ } else {
+ SENT_FAIL_COUNTER.incrementAndGet();
+ }
+ }
+
+ @Override
+ public void onException(Throwable e) {
+ TOTAL_COUNTER.incrementAndGet();
+ SENT_EXCEPT_COUNTER.incrementAndGet();
+ logger.error("Send message error!", e);
+ }
+ }
+
+ private static class TupleValue {
+ public String topic = null;
+ public String filter = null;
+
+ public TupleValue(String topic) {
+ this.topic = topic;
+ }
+
+ public TupleValue(String topic, String filter) {
+ this.topic = topic;
+ this.filter = filter;
+ }
+
+ }
+
+ public static void main(String[] args) {
+ CliProducer cliProducer = new CliProducer();
+ try {
+ boolean result = cliProducer.parseParams(args);
+ if (!result) {
+ throw new Exception("Parse parameters failure!");
+ }
+ cliProducer.initTask();
+ ThreadUtils.sleep(1000);
+ while (cliProducer.msgCount < 0
+ || TOTAL_COUNTER.get() < cliProducer.msgCount * cliProducer.clientCount) {
+ ThreadUtils.sleep(cliProducer.printIntervalMs);
+ System.out.println("Required send count VS sent message count = "
+ + (cliProducer.msgCount * cliProducer.clientCount)
+ + " : " + TOTAL_COUNTER.get()
+ + " (" + SENT_SUCC_COUNTER.get()
+ + ":" + SENT_FAIL_COUNTER.get()
+ + ":" + SENT_EXCEPT_COUNTER.get()
+ + ")");
+ }
+ cliProducer.shutdown();
+ System.out.println("Finished, required send count VS sent message count = "
+ + (cliProducer.msgCount * cliProducer.clientCount)
+ + " : " + TOTAL_COUNTER.get()
+ + " (" + SENT_SUCC_COUNTER.get()
+ + ":" + SENT_FAIL_COUNTER.get()
+ + ":" + SENT_EXCEPT_COUNTER.get()
+ + ")");
+ } catch (Throwable ex) {
+ ex.printStackTrace();
+ logger.error(ex.getMessage());
+ cliProducer.help();
+ }
+
+ }
+
+
+}
[incubator-tubemq] 24/49: [TUBEMQ-485]Add the batch setting API of
consume group offset
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 1b854fb413d1226572343627cf52a8e40d0410f4
Author: gosonzhang <go...@tencent.com>
AuthorDate: Mon Jan 4 19:39:55 2021 +0800
[TUBEMQ-485]Add the batch setting API of consume group offset
---
.../org/apache/tubemq/corebase/utils/Tuple3.java | 48 +++++
.../server/broker/offset/DefaultOffsetManager.java | 83 +++-----
.../tubemq/server/broker/offset/OffsetService.java | 7 +-
.../server/broker/web/BrokerAdminServlet.java | 228 ++++++++++++++++++++-
.../tubemq/server/common/fielddef/WebFieldDef.java | 8 +-
.../server/common/utils/WebParameterUtils.java | 71 +++++++
.../tubemq/server/common/webbase/WebFieldType.java | 3 +-
7 files changed, 388 insertions(+), 60 deletions(-)
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java
new file mode 100644
index 0000000..a2d98c3
--- /dev/null
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/Tuple3.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.corebase.utils;
+
+public class Tuple3<T0, T1, T2> {
+
+ /** Field 0 of the tuple. */
+ public T0 f0 = null;
+ /** Field 1 of the tuple. */
+ public T1 f1 = null;
+ /** Field 2 of the tuple. */
+ public T2 f2 = null;
+
+ /**
+ * Creates a new tuple where all fields are null.
+ */
+ public Tuple3() {
+
+ }
+
+ /**
+ * Creates a new tuple and assigns the given values to the tuple's fields.
+ *
+ * @param value0 The value for field 0
+ * @param value1 The value for field 1
+ * @param value2 The value for field 2
+ */
+ public Tuple3(T0 value0, T1 value1, T2 value2) {
+ this.f0 = value0;
+ this.f1 = value1;
+ this.f2 = value2;
+ }
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
index df3afc4..84dabb2 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/DefaultOffsetManager.java
@@ -19,6 +19,7 @@ package org.apache.tubemq.server.broker.offset;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -26,9 +27,9 @@ import org.apache.tubemq.corebase.TBaseConstants;
import org.apache.tubemq.corebase.daemon.AbstractDaemonService;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corebase.utils.Tuple2;
+import org.apache.tubemq.corebase.utils.Tuple3;
import org.apache.tubemq.server.broker.BrokerConfig;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
-import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.broker.utils.DataStoreUtils;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorage;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorageInfo;
@@ -119,8 +120,8 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
|| (readStatus == TBaseConstants.CONSUME_MODEL_READ_FROM_MAX_ALWAYS)) {
long adjOffset = indexMaxOffset;
if (readStatus != TBaseConstants.CONSUME_MODEL_READ_FROM_MAX_ALWAYS) {
- adjOffset = reqOffset > indexMaxOffset ? indexMaxOffset : reqOffset;
- adjOffset = adjOffset < indexMinOffset ? indexMinOffset : adjOffset;
+ adjOffset = Math.min(reqOffset, indexMaxOffset);
+ adjOffset = Math.max(adjOffset, indexMinOffset);
}
regInfo.getAndSetOffset(adjOffset);
}
@@ -288,7 +289,7 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
long firstOffset = store.getIndexMinOffset();
long lastOffset = store.getIndexMaxOffset();
reSetOffset = reSetOffset < firstOffset
- ? firstOffset : reSetOffset > lastOffset ? lastOffset : reSetOffset;
+ ? firstOffset : Math.min(reSetOffset, lastOffset);
String offsetCacheKey = getOffsetCacheKey(topic, partitionId);
getAndResetTmpOffset(group, offsetCacheKey);
OffsetStorageInfo regInfo =
@@ -449,70 +450,46 @@ public class DefaultOffsetManager extends AbstractDaemonService implements Offse
return result;
}
-
/***
* Reset offset.
*
- * @param storeManager
* @param groups
- * @param topicPartOffsetMap
+ * @param topicPartOffsets
* @param modifier
* @return at least one record modified
*/
@Override
- public boolean modifyGroupOffset(
- MessageStoreManager storeManager, Set<String> groups,
- Map<String, Map<Integer, Tuple2<Long, Long>>> topicPartOffsetMap, String modifier) {
+ public boolean modifyGroupOffset(Set<String> groups,
+ List<Tuple3<String, Integer, Long>> topicPartOffsets,
+ String modifier) {
long oldOffset = -1;
- long reSetOffset = -1;
boolean changed = false;
- MessageStore store = null;
+ String offsetCacheKey = null;
StringBuilder strBuidler = new StringBuilder(512);
// set offset by group
for (String group : groups) {
- for (Map.Entry<String, Map<Integer, Tuple2<Long, Long>>> entry
- : topicPartOffsetMap.entrySet()) {
- Map<Integer, Tuple2<Long, Long>> partOffsetMap = entry.getValue();
- if (partOffsetMap == null) {
+ for (Tuple3<String, Integer, Long> tuple3 : topicPartOffsets) {
+ if (tuple3 == null
+ || tuple3.f0 == null
+ || tuple3.f1 == null
+ || tuple3.f2 == null) {
continue;
}
- // set offset
- for (Map.Entry<Integer, Tuple2<Long, Long>> entry1 : partOffsetMap.entrySet()) {
- if (entry1.getValue() == null) {
- continue;
- }
- Tuple2<Long, Long> offsetTuple = entry1.getValue();
- // get topic store
- try {
- store = storeManager.getOrCreateMessageStore(
- entry.getKey(), entry1.getKey());
- } catch (Throwable e) {
- //
- }
- if (store == null) {
- continue;
- }
- long firstOffset = store.getIndexMinOffset();
- long lastOffset = store.getIndexMaxOffset();
- // adjust reseted offset value
- reSetOffset = offsetTuple.f0 < firstOffset
- ? firstOffset : Math.min(offsetTuple.f0, lastOffset);
- String offsetCacheKey =
- getOffsetCacheKey(entry.getKey(), entry1.getKey());
- getAndResetTmpOffset(group, offsetCacheKey);
- OffsetStorageInfo regInfo = loadOrCreateOffset(group,
- entry.getKey(), entry1.getKey(), offsetCacheKey, 0);
- oldOffset = regInfo.getAndSetOffset(reSetOffset);
- changed = true;
- logger.info(strBuidler
- .append("[Offset Manager] Update offset by modifier=")
- .append(modifier).append(",reset offset=").append(reSetOffset)
- .append(",old offset=").append(oldOffset)
- .append(",updated offset=").append(regInfo.getOffset())
- .append(",group=").append(group)
- .append(",topic-partId=").append(offsetCacheKey).toString());
- strBuidler.delete(0, strBuidler.length());
- }
+ // set offset value
+ offsetCacheKey = getOffsetCacheKey(tuple3.f0, tuple3.f1);
+ getAndResetTmpOffset(group, offsetCacheKey);
+ OffsetStorageInfo regInfo = loadOrCreateOffset(group,
+ tuple3.f0, tuple3.f1, offsetCacheKey, 0);
+ oldOffset = regInfo.getAndSetOffset(tuple3.f2);
+ changed = true;
+ logger.info(strBuidler
+ .append("[Offset Manager] Update offset by modifier=")
+ .append(modifier).append(",reset offset=").append(tuple3.f2)
+ .append(",old offset=").append(oldOffset)
+ .append(",updated offset=").append(regInfo.getOffset())
+ .append(",group=").append(group)
+ .append(",topic-partId=").append(offsetCacheKey).toString());
+ strBuidler.delete(0, strBuidler.length());
}
}
return changed;
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
index fcebdfc..9dcd29a 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/offset/OffsetService.java
@@ -17,12 +17,13 @@
package org.apache.tubemq.server.broker.offset;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.tubemq.corebase.utils.Tuple2;
+import org.apache.tubemq.corebase.utils.Tuple3;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
-import org.apache.tubemq.server.broker.msgstore.MessageStoreManager;
import org.apache.tubemq.server.common.offsetstorage.OffsetStorageInfo;
@@ -68,7 +69,7 @@ public interface OffsetService {
Map<String, Map<Integer, Tuple2<Long, Long>>> queryGroupOffset(
String group, Map<String, Set<Integer>> topicPartMap);
- boolean modifyGroupOffset(MessageStoreManager storeManager, Set<String> groups,
- Map<String, Map<Integer, Tuple2<Long, Long>>> topicPartOffsetMap,
+ boolean modifyGroupOffset(Set<String> groups,
+ List<Tuple3<String, Integer, Long>> topicPartOffsets,
String modifier);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index d8f85d4..c76a6b7 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -17,6 +17,7 @@
package org.apache.tubemq.server.broker.web;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -25,9 +26,11 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.servlet.http.HttpServletRequest;
+
import org.apache.tubemq.corebase.TokenConstants;
import org.apache.tubemq.corebase.utils.TStringUtils;
import org.apache.tubemq.corebase.utils.Tuple2;
+import org.apache.tubemq.corebase.utils.Tuple3;
import org.apache.tubemq.server.broker.TubeBroker;
import org.apache.tubemq.server.broker.metadata.TopicMetadata;
import org.apache.tubemq.server.broker.msgstore.MessageStore;
@@ -89,6 +92,9 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// clone consumer group's offset from source to target
innRegisterWebMethod("admin_clone_offset",
"adminCloneGroupOffSet");
+ // set or update group's offset info
+ innRegisterWebMethod("admin_set_offset",
+ "adminSetGroupOffSet");
}
public void adminQueryAllMethods(HttpServletRequest req,
@@ -759,6 +765,74 @@ public class BrokerAdminServlet extends AbstractWebHandler {
}
/***
+ * Add or Modify consumer group offset.
+ *
+ * @param req
+ * @param sBuilder process result
+ */
+ public void adminSetGroupOffSet(HttpServletRequest req,
+ StringBuilder sBuilder) {
+ // get group list
+ ProcessResult result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSGROUPNAME, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ Set<String> groupNameSet = (Set<String>) result.retData1;
+ // get set mode
+ result = WebParameterUtils.getBooleanParamValue(req,
+ WebFieldDef.MANUALSET, true, false);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ boolean manualSet = (Boolean) result.retData1;
+ // get modify user
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.MODIFYUSER, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ List<Tuple3<String, Integer, Long>> resetOffsets;
+ final String modifier = (String) result.retData1;
+ if (manualSet) {
+ // get offset json info
+ result = WebParameterUtils.getJsonDictParamValue(req,
+ WebFieldDef.OFFSETJSON, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ Map<String, Long> manOffsets =
+ (Map<String, Long>) result.retData1;
+ // valid and transfer offset format
+ result = validManOffsetResetInfo(WebFieldDef.OFFSETJSON, manOffsets);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ resetOffsets =
+ (List<Tuple3<String, Integer, Long>>) result.retData1;
+ } else {
+ // get the topic set to be set
+ result = WebParameterUtils.getStringParamValue(req,
+ WebFieldDef.COMPSTOPICNAME, true, null);
+ if (!result.success) {
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
+ }
+ Set<String> topicSet = (Set<String>) result.retData1;
+ // transfer offset format
+ resetOffsets = buildOffsetResetInfo(topicSet);
+ }
+ boolean changed = broker.getOffsetManager().modifyGroupOffset(
+ groupNameSet, resetOffsets, modifier);
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
+ }
+
+ /***
* Clone consume group offset, clone A group's offset to other group.
*
* @param req
@@ -821,12 +895,162 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// query offset from source group
Map<String, Map<Integer, Tuple2<Long, Long>>> srcGroupOffsets =
broker.getOffsetManager().queryGroupOffset(srcGroupName, topicPartMap);
+ // transfer offset format
+ List<Tuple3<String, Integer, Long>> resetOffsets =
+ buildOffsetResetInfo(srcGroupOffsets);
boolean changed = broker.getOffsetManager().modifyGroupOffset(
- broker.getStoreManager(), tgtGroupNameSet, srcGroupOffsets, modifier);
+ tgtGroupNameSet, resetOffsets, modifier);
// builder return result
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"OK\"}");
}
+ // build reset offset info
+ private List<Tuple3<String, Integer, Long>> buildOffsetResetInfo(
+ Map<String, Map<Integer, Tuple2<Long, Long>>> topicPartOffsetMap) {
+ long adjOffset = -1;
+ MessageStore store = null;
+ List<Tuple3<String, Integer, Long>> result = new ArrayList<>();
+ MessageStoreManager storeManager = broker.getStoreManager();
+ for (Map.Entry<String, Map<Integer, Tuple2<Long, Long>>> entry
+ : topicPartOffsetMap.entrySet()) {
+ Map<Integer, Tuple2<Long, Long>> partOffsetMap = entry.getValue();
+ if (partOffsetMap == null) {
+ continue;
+ }
+ // process offset value
+ for (Map.Entry<Integer, Tuple2<Long, Long>> entry1 : partOffsetMap.entrySet()) {
+ if (entry1.getValue() == null) {
+ continue;
+ }
+ Tuple2<Long, Long> offsetTuple = entry1.getValue();
+ // get topic store
+ try {
+ store = storeManager.getOrCreateMessageStore(
+ entry.getKey(), entry1.getKey());
+ } catch (Throwable e) {
+ //
+ }
+ if (store == null) {
+ continue;
+ }
+ long firstOffset = store.getIndexMinOffset();
+ long lastOffset = store.getIndexMaxOffset();
+ // adjust reset offset value
+ adjOffset = offsetTuple.f0 < firstOffset
+ ? firstOffset : Math.min(offsetTuple.f0, lastOffset);
+ result.add(new Tuple3<>(entry.getKey(), entry1.getKey(), adjOffset));
+ }
+ }
+ return result;
+ }
+
+ // build reset offset info
+ private List<Tuple3<String, Integer, Long>> buildOffsetResetInfo(Set<String> topicSet) {
+ MessageStore store = null;
+ List<Tuple3<String, Integer, Long>> result = new ArrayList<>();
+ MessageStoreManager storeManager = broker.getStoreManager();
+ // get topic's partition set
+ Map<String, Set<Integer>> topicPartMap =
+ validAndGetPartitions(null, topicSet);
+ // fill current topic's max offset value
+ for (Map.Entry<String, Set<Integer>> entry : topicPartMap.entrySet()) {
+ if (entry.getKey() == null
+ || entry.getValue() == null
+ || entry.getValue().isEmpty()) {
+ continue;
+ }
+ Set<Integer> partitionSet = entry.getValue();
+ for (Integer partId : partitionSet) {
+ // get topic store
+ try {
+ store = storeManager.getOrCreateMessageStore(
+ entry.getKey(), partId);
+ } catch (Throwable e) {
+ //
+ }
+ if (store == null) {
+ continue;
+ }
+ result.add(new Tuple3<>(entry.getKey(),
+ partId, store.getIndexMaxOffset()));
+ }
+ }
+ return result;
+ }
+
+ // build reset offset info
+ private ProcessResult validManOffsetResetInfo(WebFieldDef fieldDef,
+ Map<String, Long> manOffsetInfoMap) {
+ String brokerId;
+ String topicName;
+ String strPartId;
+ int partitionId;
+ long adjOffset;
+ MessageStore store = null;
+ ProcessResult procResult = new ProcessResult();
+ MessageStoreManager storeManager = broker.getStoreManager();
+ List<Tuple3<String, Integer, Long>> offsetVals = new ArrayList<>();
+ String localBrokerId = String.valueOf(broker.getTubeConfig().getBrokerId());
+ // get topic configure infos
+ Map<String, TopicMetadata> topicConfigMap =
+ broker.getMetadataManager().getTopicConfigMap();
+ for (Map.Entry<String, Long> entry : manOffsetInfoMap.entrySet()) {
+ if (entry.getKey() == null || entry.getValue() == null) {
+ continue;
+ }
+ // parse and check partitionKey value
+ String[] keyItems = entry.getKey().split(TokenConstants.ATTR_SEP);
+ if (keyItems.length != 3) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name).append("'s key invalid:")
+ .append(entry.getKey())
+ .append(" must be brokerId:topicName:partitionId !").toString());
+ return procResult;
+ }
+ brokerId = keyItems[0].trim();
+ topicName = keyItems[1].trim();
+ strPartId = keyItems[2].trim();
+ if (!localBrokerId.equals(brokerId)
+ || !topicConfigMap.containsKey(topicName)) {
+ continue;
+ }
+ try {
+ partitionId = Integer.parseInt(strPartId);
+ } catch (NumberFormatException e) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name).append("'s key invalid:")
+ .append(entry.getKey())
+ .append("'s partitionId value not number!").toString());
+ return procResult;
+ }
+ // check and adjust offset value
+ try {
+ store = storeManager.getOrCreateMessageStore(topicName, partitionId);
+ } catch (Throwable e) {
+ //
+ }
+ if (store == null) {
+ continue;
+ }
+ long firstOffset = store.getIndexMinOffset();
+ long lastOffset = store.getIndexMaxOffset();
+ adjOffset = entry.getValue() < firstOffset
+ ? firstOffset : Math.min(entry.getValue(), lastOffset);
+ offsetVals.add(new Tuple3<>(topicName, partitionId, adjOffset));
+ }
+ if (offsetVals.isEmpty()) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append("'s value is invalid!").toString());
+ } else {
+ procResult.setSuccResult(offsetVals);
+ }
+ return procResult;
+ }
+
// builder group's offset info
private Map<String, Map<String, Map<Integer, GroupOffsetInfo>>> getGroupOffsetInfo(
Set<String> groupSet, Set<String> topicSet) {
@@ -872,7 +1096,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
private Map<String, Set<Integer>> validAndGetPartitions(String group, Set<String> topicSet) {
Map<String, Set<Integer>> topicPartMap = new HashMap<>();
// query stored topic set stored in memory or zk
- if (topicSet.isEmpty()) {
+ if (topicSet.isEmpty() && group != null) {
topicSet = broker.getOffsetManager().getGroupSubInfo(group);
}
// get topic's partitionIds
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index ec97421..45b862d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -78,7 +78,13 @@ public enum WebFieldDef {
RegexDef.TMP_GROUP),
TGTCOMPSGROUPNAME(19, "targetGroupName", "tgtGroup",
WebFieldType.COMPSTRING, "Offset clone target group name",
- TBaseConstants.META_MAX_GROUPNAME_LENGTH, RegexDef.TMP_GROUP);
+ TBaseConstants.META_MAX_GROUPNAME_LENGTH, RegexDef.TMP_GROUP),
+ MANUALSET(20, "manualSet", "manSet",
+ WebFieldType.BOOLEAN, "Whether manual offset setting mode"),
+ OFFSETJSON(21, "offsetJsonSet", "offsetSet",
+ WebFieldType.JSONTYPE, "The offset set that needs to be added or modified");
+
+
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
index 1202d33..fddd5de 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/WebParameterUtils.java
@@ -478,6 +478,77 @@ public class WebParameterUtils {
}
/**
+ * Parse the parameter value from an json dict
+ *
+ * @param req Http Servlet Request
+ * @param fieldDef the parameter field definition
+ * @param required a boolean value represent whether the parameter is must required
+ * @param defValue a default value returned if failed to parse value from the given object
+ * @return valid result for the parameter value
+ */
+ public static ProcessResult getJsonDictParamValue(HttpServletRequest req,
+ WebFieldDef fieldDef,
+ boolean required,
+ Map<String, Long> defValue) {
+ ProcessResult procResult = new ProcessResult();
+ // get parameter value
+ String paramValue = req.getParameter(fieldDef.name);
+ if (paramValue == null) {
+ paramValue = req.getParameter(fieldDef.shortName);
+ }
+ if (TStringUtils.isNotBlank(paramValue)) {
+ // Cleanup value extra characters
+ paramValue = escDoubleQuotes(paramValue.trim());
+ }
+ // Check if the parameter exists
+ if (TStringUtils.isBlank(paramValue)) {
+ if (required) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append(" is missing or value is null or blank!").toString());
+ } else {
+ procResult.setSuccResult(defValue);
+ }
+ return procResult;
+ }
+ try {
+ paramValue = URLDecoder.decode(paramValue,
+ TBaseConstants.META_DEFAULT_CHARSET_NAME);
+ } catch (UnsupportedEncodingException e) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append(" decode error, exception is ")
+ .append(e.toString()).toString());
+ }
+ if (TStringUtils.isBlank(paramValue)) {
+ if (required) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append("'s value is blank!").toString());
+ } else {
+ procResult.setSuccResult(defValue);
+ }
+ return procResult;
+ }
+ if (fieldDef.valMaxLen != TBaseConstants.META_VALUE_UNDEFINED) {
+ if (paramValue.length() > fieldDef.valMaxLen) {
+ procResult.setFailResult(fieldDef.id,
+ new StringBuilder(512).append("Parameter ")
+ .append(fieldDef.name)
+ .append("'s length over max allowed length (")
+ .append(fieldDef.valMaxLen).append(")!").toString());
+ return procResult;
+ }
+ }
+ procResult.setSuccResult(new Gson().fromJson(paramValue,
+ new TypeToken<Map<String, Long>>(){}.getType()));
+ return procResult;
+ }
+
+ /**
* process string default value
*
* @param procResult process result
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
index b83a966..2f32cb1 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebFieldType.java
@@ -28,7 +28,8 @@ public enum WebFieldType {
BOOLEAN(4, "Boolean"),
DATE(5, "Date"),
COMPSTRING(6, "Compound string"),
- COMPINT(7, "Compound integer");
+ COMPINT(7, "Compound integer"),
+ JSONTYPE(8, "Json");
public int value;
[incubator-tubemq] 15/49: [TUBEMQ-450]TubeClientException: Generate
producer id failed (#351)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit ea22d0b4407084412de82fb6f7e2f8603243347e
Author: gosonzhang <46...@qq.com>
AuthorDate: Wed Dec 16 09:43:32 2020 +0800
[TUBEMQ-450]TubeClientException: Generate producer id failed (#351)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../tubemq/client/config/TubeClientConfig.java | 7 ++
.../apache/tubemq/corebase/utils/AddressUtils.java | 104 +++++++++++++--------
2 files changed, 71 insertions(+), 40 deletions(-)
diff --git a/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java b/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java
index 32c0a6f..b1bd3b1 100644
--- a/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java
+++ b/tubemq-client/src/main/java/org/apache/tubemq/client/config/TubeClientConfig.java
@@ -546,6 +546,12 @@ public class TubeClientConfig {
public String toJsonString() {
int num = 0;
+ String localAddress = null;
+ try {
+ localAddress = AddressUtils.getLocalAddress();
+ } catch (Throwable e) {
+ //
+ }
StringBuilder sBuilder = new StringBuilder(512);
sBuilder.append("{\"masterInfo\":[");
for (String item : this.masterInfo.getAddrMap4Failover().keySet()) {
@@ -580,6 +586,7 @@ public class TubeClientConfig {
.append(",\"enableUserAuthentic\":").append(this.enableUserAuthentic)
.append(",\"usrName\":\"").append(this.usrName)
.append("\",\"usrPassWord\":\"").append(this.usrPassWord)
+ .append("\",\"localAddress\":\"").append(localAddress)
.append("\",").append(this.tlsConfig.toString())
.append("}").toString();
}
diff --git a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
index 206cb93..5a76af2 100644
--- a/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
+++ b/tubemq-core/src/main/java/org/apache/tubemq/corebase/utils/AddressUtils.java
@@ -51,35 +51,31 @@ public class AddressUtils {
return checkValidIp(allInterface, currLocalHost);
}
- private static boolean checkValidIp(Enumeration<NetworkInterface> allInterface, String currLocalHost) {
- String localIp;
- try {
- while (allInterface.hasMoreElements()) {
- NetworkInterface oneInterface = allInterface.nextElement();
- if (oneInterface == null
- || oneInterface.isLoopback()
- || !oneInterface.isUp()) {
- continue;
+ private static boolean checkValidIp(Enumeration<NetworkInterface> allInterface,
+ String currLocalHost) {
+ String fstV4IP = null;
+ while (allInterface.hasMoreElements()) {
+ try {
+ Tuple2<Boolean, String> result =
+ getValidIPV4Address(allInterface.nextElement(), currLocalHost);
+ if (result.f0) {
+ localIPAddress = currLocalHost;
+ return true;
}
- Enumeration<InetAddress> allAddress = oneInterface.getInetAddresses();
- while (allAddress.hasMoreElements()) {
- InetAddress oneAddress = allAddress.nextElement();
- localIp = oneAddress.getHostAddress();
- if (TStringUtils.isBlank(localIp)
- || "127.0.0.1".equals(localIp)) {
- continue;
- }
- if (localIp.equals(currLocalHost)) {
- localIPAddress = localIp;
- return true;
- }
+ if (TStringUtils.isEmpty(fstV4IP)) {
+ fstV4IP = result.f1;
}
+ } catch (Throwable e) {
+ //
}
- } catch (SocketException e) {
- throw new AddressException("error get local ip, ex {}", e);
}
- throw new AddressException(new StringBuilder(256).append("Illegal parameter: not found the ip(")
- .append(currLocalHost).append(") in local networkInterfaces!").toString());
+ if (fstV4IP != null) {
+ localIPAddress = fstV4IP;
+ return true;
+ }
+ throw new AddressException(new StringBuilder(256)
+ .append("Illegal parameter: not found the ip(").append(currLocalHost)
+ .append(") or ip v4 address in local networkInterfaces!").toString());
}
public static int ipToInt(String ipAddr) {
@@ -155,8 +151,10 @@ public class AddressUtils {
}
while (enumeration.hasMoreElements()) {
try {
- tmpAdress = getValidIPV4Address(enumeration.nextElement());
- if (tmpAdress != null) {
+ Tuple2<Boolean, String> result =
+ getValidIPV4Address(enumeration.nextElement(), null);
+ if (result.f0) {
+ tmpAdress = result.f1;
break;
}
} catch (Throwable e) {
@@ -180,9 +178,9 @@ public class AddressUtils {
public static String getIPV4LocalAddress(String defEthName) {
boolean foundNetInter = false;
- String tmpAdress = null;
try {
- Enumeration<NetworkInterface> enumeration = NetworkInterface.getNetworkInterfaces();
+ Enumeration<NetworkInterface> enumeration =
+ NetworkInterface.getNetworkInterfaces();
if (enumeration == null) {
throw new AddressException("Get NetworkInterfaces is null");
}
@@ -196,10 +194,11 @@ public class AddressUtils {
}
foundNetInter = true;
try {
- tmpAdress = getValidIPV4Address(oneInterface);
- if (tmpAdress != null) {
- localIPAddress = tmpAdress;
- return tmpAdress;
+ Tuple2<Boolean, String> result =
+ getValidIPV4Address(oneInterface, null);
+ if (result.f0) {
+ localIPAddress = result.f1;
+ return localIPAddress;
}
} catch (Throwable e) {
//
@@ -219,14 +218,28 @@ public class AddressUtils {
}
}
- public static String getValidIPV4Address(NetworkInterface networkInterface) {
+ /**
+ * get valid IPV4 address from networkInterface.
+ *
+ * @param networkInterface need check networkInterface
+ * @param checkIp The IP address to be searched,
+ * if not specified, set to null
+ * @return Search result, field 0 indicates whether it is successful,
+ * field 1 carries the matched IP value;
+ * if the checkIp is specified but not found the IP,
+ * field 1 will return the first IPV4 address
+ * @throws AddressException throw exception if found no ipv4 address
+ */
+ public static Tuple2<Boolean, String> getValidIPV4Address(
+ NetworkInterface networkInterface, String checkIp) {
try {
if (networkInterface == null ||
!networkInterface.isUp() ||
networkInterface.isLoopback() ||
"docker0".equals(networkInterface.getName())) {
- return null;
+ return new Tuple2<>(false, null);
}
+ String fstV4IP = null;
Enumeration<InetAddress> addrs = networkInterface.getInetAddresses();
while (addrs.hasMoreElements()) {
InetAddress address = addrs.nextElement();
@@ -236,16 +249,27 @@ public class AddressUtils {
continue;
}
String localIP = address.getHostAddress();
- if (TStringUtils.isEmpty(localIP) || localIP.startsWith("127.0")) {
+ if (TStringUtils.isEmpty(localIP)
+ || localIP.startsWith("127.0")) {
+ continue;
+ }
+ if (!TStringUtils.isEmpty(checkIp)) {
+ if (TStringUtils.isEmpty(fstV4IP)) {
+ fstV4IP = localIP;
+ }
+ if (localIP.equals(checkIp)) {
+ return new Tuple2<>(true, localIP);
+ }
continue;
}
- return localIP;
+ return new Tuple2<>(true, localIP);
}
- return null;
+ return new Tuple2<>(false, fstV4IP);
} catch (Throwable e) {
throw new AddressException(new StringBuilder(256)
- .append("Illegal parameter: ").append("unable to obtain valid IP from network card ")
- .append(networkInterface).toString(), e);
+ .append("Illegal parameter: ")
+ .append("unable to obtain valid IP from network card ")
+ .append(networkInterface).toString(), e);
}
}
[incubator-tubemq] 12/49: [TUBEMQ-447] Add Broker-Admin Cli (#347)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 95654835a0de3326f2842a764356b4c0e3b32022
Author: gosonzhang <46...@qq.com>
AuthorDate: Fri Dec 11 19:15:15 2020 +0800
[TUBEMQ-447] Add Broker-Admin Cli (#347)
Co-authored-by: gosonzhang <go...@tencent.com>
---
bin/tubemq-broker-admin.sh | 40 ++++++++
pom.xml | 15 +++
tubemq-server/pom.xml | 6 +-
.../server/broker/web/AbstractWebHandler.java | 8 +-
.../server/broker/web/BrokerAdminServlet.java | 19 ++++
.../tubemq/server/common/fielddef/CliArgDef.java | 20 ++--
.../tubemq/server/common/fielddef/WebFieldDef.java | 12 ++-
.../tubemq/server/common/utils/HttpUtils.java | 112 ++++++++++++++++++++
.../server/common/webbase/WebMethodMapper.java | 9 ++
.../server/master/web/action/screen/Webapi.java | 6 +-
.../tubemq/server/tools/cli/CliAbstractBase.java | 2 +-
.../tubemq/server/tools/cli/CliBrokerAdmin.java | 113 +++++++++++++++++++++
.../tubemq/server/tools/cli/CliConsumer.java | 6 +-
.../tubemq/server/tools/cli/CliProducer.java | 6 +-
14 files changed, 355 insertions(+), 19 deletions(-)
diff --git a/bin/tubemq-broker-admin.sh b/bin/tubemq-broker-admin.sh
new file mode 100644
index 0000000..b1cedf5
--- /dev/null
+++ b/bin/tubemq-broker-admin.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "$BASE_DIR" ] ; then
+ PRG="$0"
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG="`dirname "$PRG"`/$link"
+ fi
+ done
+ BASE_DIR=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ BASE_DIR=`cd "$BASE_DIR" && pwd`
+ #echo "TubeMQ master is at $BASE_DIR"
+fi
+source $BASE_DIR/bin/env.sh
+$JAVA $TOOLS_ARGS org.apache.tubemq.server.tools.cli.CliBrokerAdmin $@
diff --git a/pom.xml b/pom.xml
index bfc2b64..f9b7342 100644
--- a/pom.xml
+++ b/pom.xml
@@ -335,6 +335,11 @@
<version>2.6</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.5.2</version>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.1</version>
@@ -436,6 +441,16 @@
<version>1.7</version>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>9.4.31.v20200723</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>9.4.31.v20200723</version>
+ </dependency>
+ <dependency>
<groupId>org.ini4j</groupId>
<artifactId>ini4j</artifactId>
<version>0.5.1</version>
diff --git a/tubemq-server/pom.xml b/tubemq-server/pom.xml
index a4b6d39..985a166 100644
--- a/tubemq-server/pom.xml
+++ b/tubemq-server/pom.xml
@@ -185,14 +185,16 @@
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
- <version>9.4.31.v20200723</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
- <version>9.4.31.v20200723</version>
</dependency>
<dependency>
<groupId>junit</groupId>
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
index b44d88c..aece762 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
@@ -17,9 +17,11 @@
package org.apache.tubemq.server.broker.web;
+import static org.apache.tubemq.server.common.webbase.WebMethodMapper.getRegisteredWebMethod;
import static org.apache.tubemq.server.common.webbase.WebMethodMapper.getWebApiRegInfo;
import static org.apache.tubemq.server.common.webbase.WebMethodMapper.registerWebMethod;
import java.io.IOException;
+import java.util.List;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -42,6 +44,10 @@ public abstract class AbstractWebHandler extends HttpServlet {
doPost(req, resp);
}
+ public List<String> getSupportedMethod() {
+ return getRegisteredWebMethod();
+ }
+
@Override
protected void doPost(HttpServletRequest req,
HttpServletResponse resp) throws IOException {
@@ -56,7 +62,7 @@ public abstract class AbstractWebHandler extends HttpServlet {
WebApiRegInfo webApiRegInfo = getWebApiRegInfo(true, method);
if (webApiRegInfo == null) {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
- .append("Unsupported method ").append(method).append("}");
+ .append("Unsupported method ").append(method).append("\"}");
} else {
strBuffer = (StringBuilder) webApiRegInfo.method.invoke(webApiRegInfo.webHandler, req);
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index ab43c2e..dfb8b2d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -68,6 +68,25 @@ public class BrokerAdminServlet extends AbstractWebHandler {
// manual set offset
innRegisterWebMethod("admin_manual_set_current_offset",
"adminManualSetCurrentOffSet");
+ // get all registered methods
+ innRegisterWebMethod("admin_get_methods",
+ "adminQueryAllMethods");
+ }
+
+ public StringBuilder adminQueryAllMethods(HttpServletRequest req) throws Exception {
+ int index = 0;
+ List<String> methods = getSupportedMethod();
+ StringBuilder sBuilder = new StringBuilder(1024);
+ sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
+ for (index = 0; index < methods.size(); index++) {
+ if (index > 0) {
+ sBuilder.append(",");
+ }
+ sBuilder.append("{\"id\":").append(index + 1)
+ .append(",\"method\":\"").append(methods.get(index)).append("\"}");
+ }
+ sBuilder.append("],\"totalCnt\":").append(index + 1).append("}");
+ return sBuilder;
}
/***
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
index abb2e2a..2e39f2f 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/CliArgDef.java
@@ -30,12 +30,12 @@ public enum CliArgDef {
MASTERSERVER(null, "master-servers",
"String: format is master1_ip:port[,master2_ip:port]",
"The master address(es) to connect to."),
- MASTERURL(null, "master-url",
- "String: format is http://master_ip:master_webport/",
- "Master Service URL to which to connect.(default: http://localhost:8080/)"),
- BROKERURL(null, "broker-url",
- "String: format is http://broker_ip:broker_webport/",
- "Broker Service URL to which to connect.(default: http://localhost:8081/)"),
+ MASTERPORTAL(null, "master-portal",
+ "String: format is master_ip:master_webport",
+ "Master Service portal to which to connect.(default: 127.0.0.1:8080)"),
+ BROKERPORTAL(null, "broker-portal",
+ "String: format is broker_ip:broker_webport",
+ "Broker Service URL to which to connect.(default: 127.0.0.1:8081)"),
MESSAGES(null, "messages",
"Long: count",
"The number of messages to send or consume, If not set, production or consumption is continual."),
@@ -95,7 +95,13 @@ public enum CliArgDef {
SYNCPRODUCE(null, "sync-produce",
"Synchronous production. (default: false)"),
WITHOUTDELAY(null, "without-delay",
- "Production without delay. (default: false)");
+ "Production without delay. (default: false)"),
+ METHOD(null, "method",
+ "String: http call method",
+ "Http call method"),
+ ADMINMETHOD(null, "show-methods",
+ "Return http's methods.");
+
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
index 1025ba0..44a6f81 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/fielddef/WebFieldDef.java
@@ -62,7 +62,11 @@ public enum WebFieldDef {
"Topic name", TBaseConstants.META_MAX_TOPICNAME_LENGTH,
RegexDef.TMP_STRING),
COMPSPARTITIONID(12, "partitionId", "pid", WebFieldType.COMPINT,
- "Partition id", RegexDef.TMP_NUMBER);
+ "Partition id", RegexDef.TMP_NUMBER),
+ CALLERIP(13, "callerIp", "cip", WebFieldType.STRING,
+ "Caller ip address", TBaseConstants.META_MAX_CLIENT_HOSTNAME_LENGTH);
+
+
@@ -86,6 +90,12 @@ public enum WebFieldDef {
}
WebFieldDef(int id, String name, String shortName, WebFieldType type,
+ String desc, int valMaxLen) {
+ this(id, name, shortName, type, desc, valMaxLen,
+ TBaseConstants.META_VALUE_UNDEFINED, false, null);
+ }
+
+ WebFieldDef(int id, String name, String shortName, WebFieldType type,
String desc, RegexDef regexDef) {
this(id, name, shortName, type, desc,
TBaseConstants.META_VALUE_UNDEFINED, regexDef);
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/HttpUtils.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/HttpUtils.java
new file mode 100644
index 0000000..b600327
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/utils/HttpUtils.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.common.utils;
+
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.apache.tubemq.corebase.utils.AddressUtils;
+import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.server.common.fielddef.WebFieldDef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+/**
+ * This class is used to process http connection and return result conversion,
+ * currently does not support https
+ */
+public class HttpUtils {
+ // log printer
+ private static final Logger logger =
+ LoggerFactory.getLogger(HttpUtils.class);
+
+
+ /* Send request to target server. */
+ public static JsonObject requestWebService(String url,
+ Map<String, String> inParamMap) throws Exception {
+ if (url == null) {
+ throw new Exception("Web service url is null!");
+ }
+ if (url.trim().toLowerCase().startsWith("https://")) {
+ throw new Exception("Unsupported https protocol!");
+ }
+ // process business parameters
+ ArrayList<BasicNameValuePair> params = new ArrayList<>();
+ if (inParamMap != null && !inParamMap.isEmpty()) {
+ for (Map.Entry<String, String> entry : inParamMap.entrySet()) {
+ params.add(new BasicNameValuePair(entry.getKey(), entry.getValue()));
+ }
+ if (inParamMap.containsKey(WebFieldDef.CALLERIP.shortName)
+ || inParamMap.containsKey(WebFieldDef.CALLERIP.name)) {
+ params.add(new BasicNameValuePair(WebFieldDef.CALLERIP.name,
+ AddressUtils.getIPV4LocalAddress()));
+ }
+ }
+ // build connect configure
+ RequestConfig requestConfig = RequestConfig.custom()
+ .setConnectTimeout(50000).setSocketTimeout(60000).build();
+ // build HttpClient and HttpPost objects
+ CloseableHttpClient httpclient = null;
+ HttpPost httpPost = null;
+ JsonObject jsonRes = null;
+ JsonParser jsonParser = new JsonParser();
+ try {
+ httpclient = HttpClients.custom()
+ .setDefaultRequestConfig(requestConfig).build();
+ httpPost = new HttpPost(url);
+ UrlEncodedFormEntity se = new UrlEncodedFormEntity(params);
+ httpPost.setEntity(se);
+ // send http request and process response
+ CloseableHttpResponse response = httpclient.execute(httpPost);
+ String returnStr = EntityUtils.toString(response.getEntity());
+ if (TStringUtils.isNotBlank(returnStr)
+ && response.getStatusLine().getStatusCode() == 200) {
+ jsonRes = jsonParser.parse(returnStr).getAsJsonObject();
+ }
+ } catch (Throwable e) {
+ throw new Exception("Connecting " + url + " throw an error!", e);
+ } finally {
+ if (httpPost != null) {
+ httpPost.releaseConnection();
+ }
+ if (httpclient != null) {
+ try {
+ httpclient.close();
+ } catch (IOException ie) {
+ logger.error("Close HttpClient error.", ie);
+ }
+ }
+ }
+ return jsonRes;
+ }
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
index a856014..83e0472 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/common/webbase/WebMethodMapper.java
@@ -18,7 +18,9 @@
package org.apache.tubemq.server.common.webbase;
import java.lang.reflect.Method;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,6 +69,13 @@ public class WebMethodMapper {
.append(webHandler.getClass().getName()).toString());
}
+ public static List<String> getRegisteredWebMethod() {
+ List<String> methods = new ArrayList<>();
+ methods.addAll(WEB_QRY_METHOD_MAP.keySet());
+ methods.addAll(WEB_MDY_METHOD_MAP.keySet());
+ return methods;
+ }
+
public static class WebApiRegInfo {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
index 5d4de04..5c5bb3f 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/web/action/screen/Webapi.java
@@ -105,7 +105,7 @@ public class Webapi implements Action {
WebMethodMapper.WebApiRegInfo webApiRegInfo = getWebApiRegInfo(isQuery, method);
if (webApiRegInfo == null) {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"Unsupported method: ")
- .append(method).append("}");
+ .append(method).append("\"}");
requestContext.put("sb", strBuffer.toString());
} else {
@@ -120,8 +120,8 @@ public class Webapi implements Action {
}
}
} catch (Throwable e) {
- strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"Bad request from client :")
- .append(e.getMessage()).append("}");
+ strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"Bad request from client, ")
+ .append(e.getMessage()).append("\"}");
requestContext.put("sb", strBuffer.toString());
}
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java
index c0903f2..0a0953d 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliAbstractBase.java
@@ -71,6 +71,6 @@ public abstract class CliAbstractBase {
protected abstract void initCommandOptions();
- public abstract boolean parseParams(String[] args) throws Exception;
+ public abstract boolean processParams(String[] args) throws Exception;
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliBrokerAdmin.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliBrokerAdmin.java
new file mode 100644
index 0000000..695ef68
--- /dev/null
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliBrokerAdmin.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tubemq.server.tools.cli;
+
+import com.google.gson.JsonObject;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.ParseException;
+import org.apache.tubemq.corebase.utils.TStringUtils;
+import org.apache.tubemq.server.common.fielddef.CliArgDef;
+import org.apache.tubemq.server.common.utils.HttpUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+
+/**
+ * This class is use to process CLI Broker Admin process.
+ *
+ *
+ */
+public class CliBrokerAdmin extends CliAbstractBase {
+
+ private static final Logger logger =
+ LoggerFactory.getLogger(CliBrokerAdmin.class);
+
+ private static final String defBrokerPortal = "127.0.0.1:8081";
+
+
+ public CliBrokerAdmin() {
+ super("tubemq-broker-admin.sh");
+ initCommandOptions();
+ }
+
+ /**
+ * Init command options
+ */
+ protected void initCommandOptions() {
+ // add the cli required parameters
+ addCommandOption(CliArgDef.BROKERPORTAL);
+ addCommandOption(CliArgDef.ADMINMETHOD);
+ addCommandOption(CliArgDef.METHOD);
+
+ }
+
+ public boolean processParams(String[] args) throws Exception {
+ // parse parameters and check value
+ CommandLine cli = parser.parse(options, args);
+ if (cli == null) {
+ throw new ParseException("Parse args failure");
+ }
+ if (cli.hasOption(CliArgDef.VERSION.longOpt)) {
+ version();
+ }
+ if (cli.hasOption(CliArgDef.HELP.longOpt)) {
+ help();
+ }
+ String brokerAddr = defBrokerPortal;
+ if (cli.hasOption(CliArgDef.BROKERPORTAL.longOpt)) {
+ brokerAddr = cli.getOptionValue(CliArgDef.BROKERPORTAL.longOpt);
+ if (TStringUtils.isBlank(brokerAddr)) {
+ throw new Exception(CliArgDef.BROKERPORTAL.longOpt + " is required!");
+ }
+ }
+ JsonObject result = null;
+ Map<String, String> inParamMap = new HashMap<>();
+ String brokerUrl = "http://" + brokerAddr + "/broker.htm";
+ if (cli.hasOption(CliArgDef.ADMINMETHOD.longOpt)) {
+ inParamMap.put(CliArgDef.METHOD.longOpt, "admin_get_methods");
+ result = HttpUtils.requestWebService(brokerUrl, inParamMap);
+ System.out.println(result.toString());
+ System.exit(0);
+ }
+ String methodStr = cli.getOptionValue(CliArgDef.METHOD.longOpt);
+ if (TStringUtils.isBlank(methodStr)) {
+ throw new Exception(CliArgDef.METHOD.longOpt + " is required!");
+ }
+ inParamMap.put(CliArgDef.METHOD.longOpt, methodStr);
+ result = HttpUtils.requestWebService(brokerUrl, inParamMap);
+ System.out.println(result.toString());
+ return true;
+ }
+
+ public static void main(String[] args) {
+ CliBrokerAdmin cliBrokerAdmin = new CliBrokerAdmin();
+ try {
+ cliBrokerAdmin.processParams(args);
+ } catch (Throwable ex) {
+ ex.printStackTrace();
+ logger.error(ex.getMessage());
+ cliBrokerAdmin.help();
+ }
+
+ }
+
+
+}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
index 359ea44..c8938a6 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliConsumer.java
@@ -90,6 +90,7 @@ public class CliConsumer extends CliAbstractBase {
/**
* Init command options
*/
+ @Override
protected void initCommandOptions() {
// add the cli required parameters
addCommandOption(CliArgDef.MASTERSERVER);
@@ -105,7 +106,8 @@ public class CliConsumer extends CliAbstractBase {
addCommandOption(CliArgDef.OUTPUTINTERVAL);
}
- public boolean parseParams(String[] args) throws Exception {
+ @Override
+ public boolean processParams(String[] args) throws Exception {
// parse parameters and check value
CommandLine cli = parser.parse(options, args);
if (cli == null) {
@@ -364,7 +366,7 @@ public class CliConsumer extends CliAbstractBase {
public static void main(String[] args) {
CliConsumer cliConsumer = new CliConsumer();
try {
- boolean result = cliConsumer.parseParams(args);
+ boolean result = cliConsumer.processParams(args);
if (!result) {
throw new Exception("Parse parameters failure!");
}
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
index 2734ac1..1b2f25a 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/tools/cli/CliProducer.java
@@ -93,6 +93,7 @@ public class CliProducer extends CliAbstractBase {
/**
* Init command options
*/
+ @Override
protected void initCommandOptions() {
// add the cli required parameters
addCommandOption(CliArgDef.MASTERSERVER);
@@ -110,7 +111,8 @@ public class CliProducer extends CliAbstractBase {
addCommandOption(CliArgDef.WITHOUTDELAY);
}
- public boolean parseParams(String[] args) throws Exception {
+ @Override
+ public boolean processParams(String[] args) throws Exception {
// parse parameters and check value
CommandLine cli = parser.parse(options, args);
if (cli == null) {
@@ -348,7 +350,7 @@ public class CliProducer extends CliAbstractBase {
public static void main(String[] args) {
CliProducer cliProducer = new CliProducer();
try {
- boolean result = cliProducer.parseParams(args);
+ boolean result = cliProducer.processParams(args);
if (!result) {
throw new Exception("Parse parameters failure!");
}
[incubator-tubemq] 48/49: [TUBEMQ-551] Adjust NOTICE file content
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 56cb40bb1b233fba42a79deaaa57d6fe402473f0
Author: gosonzhang <go...@tencent.com>
AuthorDate: Thu Jan 28 20:06:36 2021 +0800
[TUBEMQ-551] Adjust NOTICE file content
---
NOTICE | 155 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 154 insertions(+), 1 deletion(-)
diff --git a/NOTICE b/NOTICE
index c5234a8..bcefb45 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,6 +1,159 @@
Apache TubeMQ
-Copyright 2019-2020 The Apache Software Foundation.
+Copyright 2019-2021 The Apache Software Foundation.
This product includes software developed at
The Apache Software Foundation (https://www.apache.org/).
+
+Apache HBase includes the following in its NOTICE file:
+
+| Apache HBase
+| Copyright 2007-2020 The Apache Software Foundation
+|
+| This product includes software developed at
+| The Apache Software Foundation (http://www.apache.org/).
+|
+| --
+| This product incorporates portions of the 'Hadoop' project
+|
+| Copyright 2007-2009 The Apache Software Foundation
+|
+| Licensed under the Apache License v2.0
+| --
+| Our Orca logo we got here: http://www.vectorfree.com/jumping-orca
+| It is licensed Creative Commons Attribution 3.0.
+| See https://creativecommons.org/licenses/by/3.0/us/
+| We changed the logo by stripping the colored background, inverting
+| it and then rotating it some.
+|
+| Later we found that vectorfree.com image is not properly licensed.
+| The original is owned by vectorportal.com. The original was
+| relicensed so we could use it as Creative Commons Attribution 3.0.
+| The license is bundled with the download available here:
+| http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp
+| --
+| This product includes portions of the Bootstrap project v3.0.0
+|
+| Copyright 2013 Twitter, Inc.
+|
+| Licensed under the Apache License v2.0
+|
+| This product uses the Glyphicons Halflings icon set.
+|
+| http://glyphicons.com/
+|
+| Copyright Jan Kovařík
+|
+| Licensed under the Apache License v2.0 as a part of the Bootstrap project.
+|
+| --
+| This product includes portions of the Guava project v14 and v21, specifically
+| 'hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java'
+| 'hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java'
+|
+| Copyright (C) 2007 The Guava Authors
+|
+| Licensed under the Apache License, Version 2.0
+| --
+| This product includes portions of Jetty project, specially
+| 'hbase-shaded-hbase-shaded-testing-util/src/main/resources/org/apache/hadoop/hbase/shaded/org/mortbay/jetty/webapp/webdefault.xml'
+
+
+Apache MINA includes the following in its NOTICE file:
+
+| Apache MINA
+| Copyright 2007-2016 The Apache Software Foundation.
+|
+| This product includes software developed at
+| The Apache Software Foundation (http://www.apache.org/).
+
+
+Apache Avro includes the following in its NOTICE file:
+
+| Apache Avro
+| Copyright 2010-2019 The Apache Software Foundation
+|
+| This product includes software developed at
+| The Apache Software Foundation (https://www.apache.org/).
+|
+| NUnit license acknowledgement:
+|
+| | Portions Copyright © 2002-2012 Charlie Poole or Copyright © 2002-2004 James
+| | W. Newkirk, Michael C. Two, Alexei A. Vorontsov or Copyright © 2000-2002
+| | Philip A. Craig
+|
+| Based upon the representations of upstream licensors, it is understood that
+| portions of the mapreduce API included in the Java implementation are licensed
+| from various contributors under one or more contributor license agreements to
+| Odiago, Inc. and were then contributed by Odiago to Apache Avro, which has now
+| made them available under the Apache 2.0 license. The original file header text
+| is:
+|
+| | Licensed to Odiago, Inc. under one or more contributor license
+| | agreements. See the NOTICE file distributed with this work for
+| | additional information regarding copyright ownership. Odiago, Inc.
+| | licenses this file to you under the Apache License, Version 2.0
+| | (the "License"); you may not use this file except in compliance
+| | with the License. You may obtain a copy of the License at
+| |
+| | https://www.apache.org/licenses/LICENSE-2.0
+| |
+| | Unless required by applicable law or agreed to in writing, software
+| | distributed under the License is distributed on an "AS IS" BASIS,
+| | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+| | implied. See the License for the specific language governing
+| | permissions and limitations under the License.
+|
+| The Odiago NOTICE at the time of the contribution:
+|
+| | This product includes software developed by Odiago, Inc.
+| | (https://www.wibidata.com).
+|
+| Apache Ivy includes the following in its NOTICE file:
+|
+| | Apache Ivy
+| | Copyright 2007-2010 The Apache Software Foundation
+| |
+| | This product includes software developed by
+| | The Apache Software Foundation (https://www.apache.org/).
+| |
+| | Portions of Ivy were originally developed by
+| | Jayasoft SARL (http://www.jayasoft.fr/)
+| | and are licensed to the Apache Software Foundation under the
+| | "Software Grant License Agreement"
+| |
+| | SSH and SFTP support is provided by the JCraft JSch package,
+| | which is open source software, available under
+| | the terms of a BSD style license.
+| | The original software and related information is available
+| | at http://www.jcraft.com/jsch/.
+|
+| Apache Log4Net includes the following in its NOTICE file:
+|
+| | Apache log4net
+| | Copyright 2004-2015 The Apache Software Foundation
+| |
+| | This product includes software developed at
+| | The Apache Software Foundation (https://www.apache.org/).
+|
+| csharp reflect serializers were contributed by Pitney Bowes Inc.
+|
+| | Copyright 2019 Pitney Bowes Inc.
+| | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
+| | You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0.
+| | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+| | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+| | See the License for the specific language governing permissions and limitations under the License.
+
+
+Apache Pulsar includes the following in its NOTICE file:
+
+| Apache Pulsar
+| Copyright 2017-2019 The Apache Software Foundation
+|
+| This product includes software developed at
+| The Apache Software Foundation (http://www.apache.org/).
+
+
+
+
[incubator-tubemq] 18/49: [TUBEMQ-463]Adjust Master rebalance
process implementation (#355)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 380436e5b802b5d3f02c316d6ea2931a0a49aaf0
Author: gosonzhang <46...@qq.com>
AuthorDate: Sun Dec 20 16:46:04 2020 +0800
[TUBEMQ-463]Adjust Master rebalance process implementation (#355)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../apache/tubemq/server/master/MasterConfig.java | 10 ++
.../org/apache/tubemq/server/master/TMaster.java | 180 ++++++++++++---------
2 files changed, 115 insertions(+), 75 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java
index 84ff2ed..b112d66 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/MasterConfig.java
@@ -79,6 +79,7 @@ public class MasterConfig extends AbstractFileConfig {
private String visitName = "";
private String visitPassword = "";
private long authValidTimeStampPeriodMs = TBaseConstants.CFG_DEFAULT_AUTH_TIMESTAMP_VALID_INTERVAL;
+ private int rebalanceParallel = 4;
/**
* getters
@@ -253,6 +254,10 @@ public class MasterConfig extends AbstractFileConfig {
return authValidTimeStampPeriodMs;
}
+ public int getRebalanceParallel() {
+ return rebalanceParallel;
+ }
+
/**
* Load file section attributes
*
@@ -460,6 +465,10 @@ public class MasterConfig extends AbstractFileConfig {
this.visitName = masterConf.get("visitName").trim();
this.visitPassword = masterConf.get("visitPassword").trim();
}
+ if (TStringUtils.isNotBlank(masterConf.get("rebalanceParallel"))) {
+ int tmpParallel = this.getInt(masterConf, "rebalanceParallel");
+ this.rebalanceParallel = (tmpParallel <= 0) ? 1 : (Math.min(tmpParallel, 20));
+ }
}
/**
@@ -606,6 +615,7 @@ public class MasterConfig extends AbstractFileConfig {
.append("useWebProxy", useWebProxy)
.append("visitName", visitName)
.append("visitPassword", visitPassword)
+ .append("rebalanceParallel", rebalanceParallel)
.append(",").append(replicationConfig.toString())
.append(",").append(tlsConfig.toString())
.append(",").append(zkConfig.toString())
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
index 8811cb0..34570de 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/master/TMaster.java
@@ -27,7 +27,10 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.commons.collections.CollectionUtils;
@@ -126,6 +129,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
new RpcServiceFactory();
private final ConsumerEventManager consumerEventManager; //consumer event manager
private final TopicPSInfoManager topicPSInfoManager; //topic publish/subscribe info manager
+ private final ExecutorService executor;
private final BrokerInfoHolder brokerHolder; //broker holder
private final ProducerInfoHolder producerHolder; //producer holder
private final ConsumerInfoHolder consumerHolder; //consumer holder
@@ -144,11 +148,10 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
private AtomicLong idGenerator = new AtomicLong(0); //id generator
private volatile boolean stopped = false; //stop flag
private Thread balancerChore; //balance chore
- private Thread resetBalancerChore; //reset balance chore
private boolean initialized = false;
private boolean startupBalance = true;
- private boolean startupResetBalance = true;
private int balanceDelayTimes = 0;
+ private AtomicInteger curBalanceParal = new AtomicInteger(0);
private Sleeper stopSleeper = new Sleeper(1000, this);
private SimpleVisitTokenManager visitTokenManager;
@@ -165,6 +168,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
this.checkAndCreateBdbDataPath();
this.masterAddInfo =
new NodeAddrInfo(masterConfig.getHostName(), masterConfig.getPort());
+ this.executor = Executors.newFixedThreadPool(this.masterConfig.getRebalanceParallel());
this.visitTokenManager = new SimpleVisitTokenManager(this.masterConfig);
this.serverAuthHandler = new SimpleCertificateMasterHandler(this.masterConfig);
this.producerHolder = new ProducerInfoHolder();
@@ -1832,7 +1836,6 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if (!this.stopped) {
Thread.sleep(masterConfig.getFirstBalanceDelayAfterStartMs());
this.balancerChore = startBalancerChore(this);
- this.resetBalancerChore = startResetBalancerChore(this);
initialized = true;
while (!this.stopped) {
stopSleeper.sleep();
@@ -1851,10 +1854,9 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
/**
* Load balance
*/
- private void balance() {
- // #lizard forgives
+ private void balance(final TMaster tMaster) {
final StringBuilder strBuffer = new StringBuilder(512);
- long rebalanceId = idGenerator.incrementAndGet();
+ final long rebalanceId = idGenerator.incrementAndGet();
if (defaultBdbStoreService != null) {
logger.info(strBuffer.append("[Rebalance Start] ").append(rebalanceId)
.append(", isMaster=").append(defaultBdbStoreService.isMaster())
@@ -1865,20 +1867,86 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
.append(", BDB service is null isMaster= false, isPrimaryNodeActive=false").toString());
}
strBuffer.delete(0, strBuffer.length());
+ int curDoingTasks = this.curBalanceParal.get();
+ if (curDoingTasks > 0) {
+ logger.info(strBuffer.append("[Rebalance End] ").append(rebalanceId)
+ .append(", the previous rebalance has ")
+ .append(curDoingTasks).append(" task(s) in progress!").toString());
+ return;
+ }
+ final boolean isStartBalance = startupBalance;
+ List<String> groupsNeedToBalance = isStartBalance ?
+ consumerHolder.getAllGroup() : getNeedToBalanceGroupList(strBuffer);
+ strBuffer.delete(0, strBuffer.length());
+ if (!groupsNeedToBalance.isEmpty()) {
+ // set parallel rebalance signal
+ curBalanceParal.set(masterConfig.getRebalanceParallel());
+ // calculate process count
+ int unitNum = (groupsNeedToBalance.size() + masterConfig.getRebalanceParallel() - 1)
+ / masterConfig.getRebalanceParallel();
+ // start processer to do reblance;
+ int startIndex = 0;
+ int endIndex = 0;
+ for (int i = 0; i < masterConfig.getRebalanceParallel(); i++) {
+ // get groups need to rebalance
+ startIndex = Math.min((i) * unitNum, groupsNeedToBalance.size());
+ endIndex = Math.min((i + 1) * unitNum, groupsNeedToBalance.size());
+ final List<String> subGroups = groupsNeedToBalance.subList(startIndex, endIndex);
+ // execute rebalance
+ this.executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (subGroups.isEmpty()) {
+ return;
+ }
+ // first process reset rebalance task;
+ try {
+ tMaster.processResetbalance(rebalanceId,
+ isStartBalance, subGroups);
+ } catch (Throwable e) {
+ logger.warn("[Rebalance processor] Error during reset-reb", e);
+ }
+ if (tMaster.isStopped()) {
+ return;
+ }
+ // second process normal rebalance task;
+ try {
+ tMaster.processRebalance(rebalanceId,
+ isStartBalance, subGroups);
+ } catch (Throwable e) {
+ logger.warn("[Rebalance processor] Error during normal-reb", e);
+ }
+ } catch (Throwable e) {
+ logger.warn("[Rebalance processor] Error during process", e);
+ } finally {
+ curBalanceParal.decrementAndGet();
+ }
+ }
+ });
+ }
+ }
+ startupBalance = false;
+ logger.info(strBuffer.append("[Rebalance End] ").append(rebalanceId).toString());
+ }
+
+ // process unReset group rebalance
+ public void processRebalance(long rebalanceId, boolean isFirstReb, List<String> groups) {
+ // #lizard forgives
Map<String, Map<String, List<Partition>>> finalSubInfoMap = null;
- if (startupBalance) {
- finalSubInfoMap =
- this.loadBalancer.bukAssign(consumerHolder, topicPSInfoManager,
- consumerHolder.getAllGroup(), defaultBrokerConfManager,
- masterConfig.getMaxGroupBrokerConsumeRate(), strBuffer);
- startupBalance = false;
+ final StringBuilder strBuffer = new StringBuilder(512);
+ // choose different load balance strategy
+ if (isFirstReb) {
+ finalSubInfoMap = this.loadBalancer.bukAssign(consumerHolder,
+ topicPSInfoManager, groups, defaultBrokerConfManager,
+ masterConfig.getMaxGroupBrokerConsumeRate(), strBuffer);
} else {
- List<String> groupsNeedToBalance = getNeedToBalanceGroupList(strBuffer);
- finalSubInfoMap =
- this.loadBalancer.balanceCluster(currentSubInfo, consumerHolder, brokerHolder,
- topicPSInfoManager, groupsNeedToBalance, defaultBrokerConfManager,
- masterConfig.getMaxGroupBrokerConsumeRate(), strBuffer);
+ finalSubInfoMap = this.loadBalancer.balanceCluster(currentSubInfo,
+ consumerHolder, brokerHolder, topicPSInfoManager, groups,
+ defaultBrokerConfManager, masterConfig.getMaxGroupBrokerConsumeRate(),
+ strBuffer);
}
+ // allocate partitions to consumers
for (Map.Entry<String, Map<String, List<Partition>>> entry : finalSubInfoMap.entrySet()) {
String consumerId = entry.getKey();
if (consumerId == null) {
@@ -1891,7 +1959,8 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
|| tupleInfo.f1 == null) {
continue;
}
- List<String> blackTopicList = this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
+ List<String> blackTopicList =
+ this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
Map<String, List<Partition>> topicSubPartMap = entry.getValue();
List<SubscribeInfo> deletedSubInfoList = new ArrayList<>();
List<SubscribeInfo> addedSubInfoList = new ArrayList<>();
@@ -1899,7 +1968,8 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
String topic = topicEntry.getKey();
List<Partition> finalPartList = topicEntry.getValue();
Map<String, Partition> currentPartMap = null;
- Map<String, Map<String, Partition>> curTopicSubInfoMap = currentSubInfo.get(consumerId);
+ Map<String, Map<String, Partition>> curTopicSubInfoMap =
+ currentSubInfo.get(consumerId);
if (curTopicSubInfoMap == null || curTopicSubInfoMap.get(topic) == null) {
currentPartMap = new HashMap<>();
} else {
@@ -1948,8 +2018,8 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
for (Partition currentPart : currentPartMap.values()) {
if ((blackTopicList.contains(currentPart.getTopic()))
|| (!finalPartList.contains(currentPart))) {
- deletedSubInfoList
- .add(new SubscribeInfo(consumerId, tupleInfo.f0, false, currentPart));
+ deletedSubInfoList.add(new SubscribeInfo(consumerId,
+ tupleInfo.f0, false, currentPart));
}
}
for (Partition finalPart : finalPartList) {
@@ -1990,42 +2060,24 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
}
}
}
- logger.info(strBuffer.append("[Rebalance End] ")
- .append(rebalanceId).toString());
}
/**
- * Reset balance
+ * process Reset balance
*/
- private void resetBalance() {
+ public void processResetbalance(long rebalanceId, boolean isFirstReb, List<String> groups) {
// #lizard forgives
- //consumer need reset offset
final StringBuilder strBuffer = new StringBuilder(512);
- long rebalanceId = idGenerator.incrementAndGet();
- if (defaultBdbStoreService != null) {
- logger.info(strBuffer.append("[ResetRebalance Start] ").append(rebalanceId)
- .append(", isMaster=").append(defaultBdbStoreService.isMaster())
- .append(", isPrimaryNodeActive=")
- .append(defaultBdbStoreService.isPrimaryNodeActive()).toString());
- } else {
- logger.info(strBuffer.append("[ResetRebalance Start] ").append(rebalanceId)
- .append(", BDB service is null isMaster= false, isPrimaryNodeActive=false").toString());
- }
- strBuffer.delete(0, strBuffer.length());
Map<String, Map<String, Map<String, Partition>>> finalSubInfoMap = null;
// choose different load balance strategy
- if (startupResetBalance) {
- finalSubInfoMap =
- this.loadBalancer.resetBukAssign(consumerHolder, topicPSInfoManager,
- consumerHolder.getAllGroup(), this.zkOffsetStorage,
- this.defaultBrokerConfManager, strBuffer);
- startupResetBalance = false;
+ if (isFirstReb) {
+ finalSubInfoMap = this.loadBalancer.resetBukAssign(consumerHolder,
+ topicPSInfoManager, groups, this.zkOffsetStorage,
+ this.defaultBrokerConfManager, strBuffer);
} else {
- List<String> groupsNeedToBalance = getNeedToBalanceGroupList(strBuffer);
- finalSubInfoMap =
- this.loadBalancer.resetBalanceCluster(currentSubInfo, consumerHolder,
- topicPSInfoManager, groupsNeedToBalance, this.zkOffsetStorage,
- this.defaultBrokerConfManager, strBuffer);
+ finalSubInfoMap = this.loadBalancer.resetBalanceCluster(currentSubInfo,
+ consumerHolder, topicPSInfoManager, groups, this.zkOffsetStorage,
+ this.defaultBrokerConfManager, strBuffer);
}
// filter
for (Map.Entry<String, Map<String, Map<String, Partition>>> entry
@@ -2041,7 +2093,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
|| tupleInfo.f1 == null) {
continue;
}
-
+ // allocate partitions to consumers
List<String> blackTopicList =
this.defaultBrokerConfManager.getBdbBlackTopicList(tupleInfo.f0);
Map<String, Map<String, Partition>> topicSubPartMap = entry.getValue();
@@ -2107,7 +2159,6 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
}
}
}
- logger.info(strBuffer.append("[ResetRebalance End] ").append(rebalanceId).toString());
}
/**
@@ -2240,9 +2291,6 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
if (this.balancerChore != null) {
this.balancerChore.interrupt();
}
- if (this.resetBalancerChore != null) {
- this.resetBalancerChore.interrupt();
- }
}
/**
@@ -2252,11 +2300,12 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
* @return
*/
private Thread startBalancerChore(final TMaster master) {
- Chore chore = new Chore("BalancerChore", masterConfig.getConsumerBalancePeriodMs(), master) {
+ Chore chore = new Chore("BalancerChore",
+ masterConfig.getConsumerBalancePeriodMs(), master) {
@Override
protected void chore() {
try {
- master.balance();
+ master.balance(master);
} catch (Throwable e) {
logger.warn("Rebalance throwable error: ", e);
}
@@ -2265,26 +2314,6 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
return ThreadUtils.setDaemonThreadRunning(chore.getThread());
}
- /**
- * Start reset balance chore
- *
- * @param master
- * @return
- */
- private Thread startResetBalancerChore(final TMaster master) {
- Chore chore = new Chore("ResetBalancerChore", masterConfig.getConsumerBalancePeriodMs(), master) {
- @Override
- protected void chore() {
- try {
- master.resetBalance();
- } catch (Throwable e) {
- logger.warn("Reset Rebalance throwable error: ", e);
- }
- }
- };
- return ThreadUtils.setDaemonThreadRunning(chore.getThread());
- }
-
public void stop() {
stop("");
}
@@ -2301,6 +2330,7 @@ public class TMaster extends HasThread implements MasterService, Stoppable {
try {
webServer.stop();
rpcServiceFactory.destroy();
+ executor.shutdown();
stopChores();
heartbeatManager.stop();
zkOffsetStorage.close();
[incubator-tubemq] 04/49: [TUBEMQ-441]An error occurred when using
the Tubemq class to create a sink table (#339)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit b2177f1c1e3242ecb3d53b917769f8aa147ee712
Author: gosonzhang <46...@qq.com>
AuthorDate: Fri Dec 4 17:36:55 2020 +0800
[TUBEMQ-441]An error occurred when using the Tubemq class to create a sink table (#339)
Co-authored-by: gosonzhang <go...@tencent.com>
---
.../org/apache/flink/connectors/tubemq/Tubemq.java | 28 ++++++++++++++++------
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/Tubemq.java b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/Tubemq.java
index 69eb279..dc265b1 100644
--- a/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/Tubemq.java
+++ b/tubemq-connectors/tubemq-connector-flink/src/main/java/org/apache/flink/connectors/tubemq/Tubemq.java
@@ -41,6 +41,9 @@ import org.apache.flink.table.descriptors.DescriptorProperties;
public class Tubemq extends ConnectorDescriptor {
@Nullable
+ private boolean consumerRole = true;
+
+ @Nullable
private String topic;
@Nullable
@@ -74,6 +77,16 @@ public class Tubemq extends ConnectorDescriptor {
}
/**
+ * Sets the client role to be used.
+ *
+ * @param isConsumer The client role if consumer.
+ */
+ public Tubemq asConsumer(boolean isConsumer) {
+ this.consumerRole = isConsumer;
+ return this;
+ }
+
+ /**
* Sets the address of tubemq master to connect.
*
* @param master The address of tubemq master.
@@ -133,13 +146,14 @@ public class Tubemq extends ConnectorDescriptor {
if (master != null) {
descriptorProperties.putString(CONNECTOR_MASTER, master);
}
-
- if (group != null) {
- descriptorProperties.putString(CONNECTOR_GROUP, group);
- }
-
- if (tids != null) {
- descriptorProperties.putString(CONNECTOR_TIDS, tids);
+ if (consumerRole) {
+ if (group != null) {
+ descriptorProperties.putString(CONNECTOR_GROUP, group);
+ }
+
+ if (tids != null) {
+ descriptorProperties.putString(CONNECTOR_TIDS, tids);
+ }
}
descriptorProperties.putPropertiesWithPrefix(CONNECTOR_PROPERTIES, properties);
[incubator-tubemq] 20/49: [TUBEMQ-472]Adjust Broker's
AbstractWebHandler class implementation
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuanbo pushed a commit to branch TUBEMQ-421
in repository https://gitbox.apache.org/repos/asf/incubator-tubemq.git
commit 4920c51d731da45371dbf57b0dc14624d561950c
Author: gosonzhang <go...@tencent.com>
AuthorDate: Tue Dec 22 19:56:49 2020 +0800
[TUBEMQ-472]Adjust Broker's AbstractWebHandler class implementation
---
.../server/broker/web/AbstractWebHandler.java | 2 +-
.../server/broker/web/BrokerAdminServlet.java | 120 +++++++++++----------
2 files changed, 64 insertions(+), 58 deletions(-)
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
index aece762..508849c 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/AbstractWebHandler.java
@@ -64,7 +64,7 @@ public abstract class AbstractWebHandler extends HttpServlet {
strBuffer.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Unsupported method ").append(method).append("\"}");
} else {
- strBuffer = (StringBuilder) webApiRegInfo.method.invoke(webApiRegInfo.webHandler, req);
+ webApiRegInfo.method.invoke(webApiRegInfo.webHandler, req, strBuffer);
}
}
} catch (Throwable e) {
diff --git a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
index dfb8b2d..1b3f524 100644
--- a/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
+++ b/tubemq-server/src/main/java/org/apache/tubemq/server/broker/web/BrokerAdminServlet.java
@@ -73,10 +73,10 @@ public class BrokerAdminServlet extends AbstractWebHandler {
"adminQueryAllMethods");
}
- public StringBuilder adminQueryAllMethods(HttpServletRequest req) throws Exception {
+ public void adminQueryAllMethods(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
int index = 0;
List<String> methods = getSupportedMethod();
- StringBuilder sBuilder = new StringBuilder(1024);
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
for (index = 0; index < methods.size(); index++) {
if (index > 0) {
@@ -86,26 +86,25 @@ public class BrokerAdminServlet extends AbstractWebHandler {
.append(",\"method\":\"").append(methods.get(index)).append("\"}");
}
sBuilder.append("],\"totalCnt\":").append(index + 1).append("}");
- return sBuilder;
}
/***
* Query broker's all consumer info.
*
* @param req
- * @return
+ * @param sBuilder process result
* @throws Exception
*/
- public StringBuilder adminQueryBrokerAllConsumerInfo(HttpServletRequest req) throws Exception {
+ public void adminQueryBrokerAllConsumerInfo(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
int index = 0;
- StringBuilder sBuilder = new StringBuilder(1024);
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.COMPSGROUPNAME, false, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
Set<String> groupNameSet = (Set<String>) result.retData1;
-
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
Map<String, ConsumerNodeInfo> map =
broker.getBrokerServiceServer().getConsumerRegisterMap();
@@ -176,23 +175,22 @@ public class BrokerAdminServlet extends AbstractWebHandler {
}
}
sBuilder.append("],\"totalCnt\":").append(index).append("}");
- return sBuilder;
}
/***
* Query broker's all message store info.
*
* @param req
- * @return
+ * @param sBuilder process result
* @throws Exception
*/
- public StringBuilder adminQueryBrokerAllMessageStoreInfo(HttpServletRequest req)
- throws Exception {
- StringBuilder sBuilder = new StringBuilder(1024);
+ public void adminQueryBrokerAllMessageStoreInfo(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.COMPSTOPICNAME, false, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
Set<String> topicNameSet = (Set<String>) result.retData1;
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"dataSet\":[");
@@ -244,28 +242,29 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("]}");
}
sBuilder.append("],\"totalCnt\":").append(recordId).append("}");
- return sBuilder;
}
/***
* Get memory store status info.
*
* @param req
- * @return
+ * @param sBuilder process result
* @throws Exception
*/
- public StringBuilder adminGetMemStoreStatisInfo(HttpServletRequest req) throws Exception {
- StringBuilder sBuilder = new StringBuilder(1024);
+ public void adminGetMemStoreStatisInfo(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.COMPSTOPICNAME, false, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
Set<String> topicNameSet = (Set<String>) result.retData1;
result = WebParameterUtils.getBooleanParamValue(req,
WebFieldDef.NEEDREFRESH, false, false);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
boolean requireRefresh = (boolean) result.retData1;
sBuilder.append("{\"result\":true,\"errCode\":0,\"errMsg\":\"Success!\",\"detail\":[");
@@ -302,46 +301,50 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("]}");
}
sBuilder.append("],\"totalCount\":").append(recordId).append("}");
- return sBuilder;
}
/***
* Manual set offset.
*
* @param req
- * @return
+ * @param sBuilder process result
* @throws Exception
*/
- public StringBuilder adminManualSetCurrentOffSet(HttpServletRequest req) throws Exception {
- StringBuilder sBuilder = new StringBuilder(512);
+ public void adminManualSetCurrentOffSet(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.TOPICNAME, true, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
final String topicName = (String) result.retData1;
result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.GROUPNAME, true, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
final String groupName = (String) result.retData1;
result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.MODIFYUSER, true, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
final String modifyUser = (String) result.retData1;
result = WebParameterUtils.getIntParamValue(req,
WebFieldDef.PARTITIONID, true, -1, 0);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
int partitionId = (Integer) result.retData1;
result = WebParameterUtils.getLongParamValue(req,
WebFieldDef.MANUALOFFSET, true, -1);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
long manualOffset = (Long) result.retData1;
List<String> topicList = broker.getMetadataManager().getTopics();
@@ -349,7 +352,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Invalid parameter: not found the topicName configure!")
.append("\"}");
- return sBuilder;
+ return;
}
MessageStoreManager storeManager = broker.getStoreManager();
MessageStore store = null;
@@ -362,21 +365,21 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Invalid parameter: not found the store by topicName!")
.append("\"}");
- return sBuilder;
+ return;
}
if (manualOffset < store.getIndexMinOffset()) {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Invalid parameter: manualOffset lower than Current MinOffset:(")
.append(manualOffset).append("<").append(store.getIndexMinOffset())
.append(")\"}");
- return sBuilder;
+ return;
}
if (manualOffset > store.getIndexMaxOffset()) {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Invalid parameter: manualOffset bigger than Current MaxOffset:(")
.append(manualOffset).append(">").append(store.getIndexMaxOffset())
.append(")\"}");
- return sBuilder;
+ return;
}
OffsetService offsetService = broker.getOffsetManager();
long oldOffset =
@@ -391,34 +394,36 @@ public class BrokerAdminServlet extends AbstractWebHandler {
.append("Manual update current Offset success!")
.append("\",\"oldOffset\":").append(oldOffset).append("}");
}
- return sBuilder;
}
/***
* Query snapshot message set.
*
* @param req
- * @return
+ * @param sBuilder process result
* @throws Exception
*/
- public StringBuilder adminQuerySnapshotMessageSet(HttpServletRequest req) throws Exception {
- StringBuilder sBuilder = new StringBuilder(1024);
+ public void adminQuerySnapshotMessageSet(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.TOPICNAME, true, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
final String topicName = (String) result.retData1;
result = WebParameterUtils.getIntParamValue(req,
WebFieldDef.PARTITIONID, true, -1, 0);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
int partitionId = (Integer) result.retData1;
result = WebParameterUtils.getIntParamValue(req,
WebFieldDef.MSGCOUNT, false, 3, 3);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
int msgCount = (Integer) result.retData1;
msgCount = Math.max(msgCount, 1);
@@ -426,52 +431,55 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Over max allowed msgCount value, allowed count is 50!")
.append("\"}");
- return sBuilder;
+ return;
}
result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.FILTERCONDS, false, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
Set<String> filterCondStrSet = (Set<String>) result.retData1;
sBuilder = broker.getBrokerServiceServer()
.getMessageSnapshot(topicName, partitionId, msgCount, filterCondStrSet, sBuilder);
- return sBuilder;
}
/***
* Query consumer group offset.
*
* @param req
- * @return
+ * @param sBuilder process result
* @throws Exception
*/
- public StringBuilder adminQueryCurrentGroupOffSet(HttpServletRequest req)
- throws Exception {
- StringBuilder sBuilder = new StringBuilder(1024);
+ public void adminQueryCurrentGroupOffSet(HttpServletRequest req,
+ StringBuilder sBuilder) throws Exception {
ProcessResult result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.TOPICNAME, true, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
final String topicName = (String) result.retData1;
result = WebParameterUtils.getStringParamValue(req,
WebFieldDef.GROUPNAME, true, null);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
final String groupName = (String) result.retData1;
result = WebParameterUtils.getIntParamValue(req,
WebFieldDef.PARTITIONID, true, -1, 0);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
int partitionId = (Integer) result.retData1;
result = WebParameterUtils.getBooleanParamValue(req,
WebFieldDef.REQUIREREALOFFSET, false, false);
if (!result.success) {
- return WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ WebParameterUtils.buildFailResult(sBuilder, result.errInfo);
+ return;
}
boolean requireRealOffset = (Boolean) result.retData1;
List<String> topicList = broker.getMetadataManager().getTopics();
@@ -479,7 +487,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Invalid parameter: not found the topicName configure!")
.append("\"}");
- return sBuilder;
+ return;
}
MessageStoreManager storeManager = broker.getStoreManager();
OffsetService offsetService = broker.getOffsetManager();
@@ -493,7 +501,7 @@ public class BrokerAdminServlet extends AbstractWebHandler {
sBuilder.append("{\"result\":false,\"errCode\":400,\"errMsg\":\"")
.append("Invalid parameter: not found the store by topicName!")
.append("\"}");
- return sBuilder;
+ return;
}
long tmpOffset = offsetService.getTmpOffset(groupName, topicName, partitionId);
long minDataOffset = store.getDataMinOffset();
@@ -528,11 +536,10 @@ public class BrokerAdminServlet extends AbstractWebHandler {
}
}
sBuilder.append("}");
- return sBuilder;
}
- public StringBuilder adminQueryConsumerRegisterInfo(HttpServletRequest req) {
- StringBuilder sBuilder = new StringBuilder(1024);
+ public void adminQueryConsumerRegisterInfo(HttpServletRequest req,
+ StringBuilder sBuilder) {
Map<String, ConsumerNodeInfo> map =
broker.getBrokerServiceServer().getConsumerRegisterMap();
int totalCnt = 0;
@@ -551,7 +558,6 @@ public class BrokerAdminServlet extends AbstractWebHandler {
.append("\",\"index\":").append(++totalCnt).append("}");
}
sBuilder.append("],\"totalCnt\":").append(totalCnt).append("}");
- return sBuilder;
}