You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by "LetLetMe (via GitHub)" <gi...@apache.org> on 2024/05/10 09:12:11 UTC
[PR] [ISSUE #8058]Support for upgrading metadata in json to rocksdb (#8045) [rocketmq]
LetLetMe opened a new pull request, #8116:
URL: https://github.com/apache/rocketmq/pull/8116
<!-- Please make sure the target branch is right. In most case, the target branch should be `develop`. -->
### Which Issue(s) This PR Fixes
<!-- Please ensure that the related issue has already been created, and [link this pull request to that issue using keywords](<https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword>) to ensure automatic closure. -->
Fixes #issue_id
### Brief Description
<!-- Write a brief description for your pull request to help the maintainer understand the reasons behind your changes. -->
### How Did You Test This Change?
<!-- In order to ensure the code quality of Apache RocketMQ, we expect every pull request to have undergone thorough testing. -->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@rocketmq.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [ISSUE #8058] Support for upgrading metadata in json to rocksdb (#8045) [rocketmq]
Posted by "lollipopjin (via GitHub)" <gi...@apache.org>.
lollipopjin commented on code in PR #8116:
URL: https://github.com/apache/rocketmq/pull/8116#discussion_r1613034293
##########
broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java:
##########
@@ -16,39 +16,116 @@
*/
package org.apache.rocketmq.broker.subscription;
-import java.io.File;
-
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.serializer.SerializerFeature;
import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.common.config.RocksDBConfigManager;
+import org.apache.rocketmq.broker.RocksDBConfigManager;
+import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
+import org.rocksdb.RocksIterator;
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
+import java.io.File;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.BiConsumer;
public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager {
+ protected RocksDBConfigManager rocksDBConfigManager;
+
public RocksDBSubscriptionGroupManager(BrokerController brokerController) {
super(brokerController, false);
- this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
+ this.rocksDBConfigManager = new RocksDBConfigManager(rocksdbConfigFilePath(), brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs());
}
@Override
public boolean load() {
- if (!this.rocksDBConfigManager.load(configFilePath(), this::decode0)) {
+ if (!rocksDBConfigManager.init()) {
+ return false;
+ }
+ if (!loadDataVersion() || !loadSubscriptionGroupAndForbidden()) {
return false;
}
this.init();
return true;
}
+ public boolean loadDataVersion() {
+ return this.rocksDBConfigManager.loadDataVersion();
+ }
+
+ public boolean loadForbidden(BiConsumer<byte[], byte[]> biConsumer) {
+ try (RocksIterator iterator = this.rocksDBConfigManager.configRocksDBStorage.forbiddenIterator()) {
+ iterator.seekToFirst();
+ while (iterator.isValid()) {
+ biConsumer.accept(iterator.key(), iterator.value());
+ iterator.next();
+ }
+ }
+ return true;
+ }
+
+ public boolean loadSubscriptionGroupAndForbidden() {
Review Comment:
If you want to do as a pipeline case here , use if-else may be a more readable way.
##########
broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java:
##########
@@ -14,46 +14,66 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.common.config;
+package org.apache.rocketmq.broker;
+import com.alibaba.fastjson.JSON;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
import org.rocksdb.FlushOptions;
import org.rocksdb.RocksIterator;
import org.rocksdb.Statistics;
import org.rocksdb.WriteBatch;
+import java.nio.charset.StandardCharsets;
import java.util.function.BiConsumer;
public class RocksDBConfigManager {
protected static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-
- protected volatile boolean isStop = false;
- protected ConfigRocksDBStorage configRocksDBStorage = null;
+ public volatile boolean isStop = false;
+ public ConfigRocksDBStorage configRocksDBStorage = null;
private FlushOptions flushOptions = null;
private volatile long lastFlushMemTableMicroSecond = 0;
+
+ private final String filePath;
private final long memTableFlushInterval;
+ private DataVersion kvDataVersion = new DataVersion();
+
- public RocksDBConfigManager(long memTableFlushInterval) {
+ public RocksDBConfigManager(String filePath, long memTableFlushInterval) {
+ this.filePath = filePath;
this.memTableFlushInterval = memTableFlushInterval;
}
- public boolean load(String configFilePath, BiConsumer<byte[], byte[]> biConsumer) {
+ public boolean init() {
this.isStop = false;
- this.configRocksDBStorage = new ConfigRocksDBStorage(configFilePath);
- if (!this.configRocksDBStorage.start()) {
- return false;
- }
- RocksIterator iterator = this.configRocksDBStorage.iterator();
+ this.configRocksDBStorage = new ConfigRocksDBStorage(filePath);
+ return this.configRocksDBStorage.start();
+ }
+ public final boolean loadDataVersion() {
+ String currDataVersionString = null;
try {
+ byte[] dataVersion = this.configRocksDBStorage.getKvDataVersion();
+ if (dataVersion != null && dataVersion.length > 0) {
+ currDataVersionString = new String(dataVersion, StandardCharsets.UTF_8);
+ }
+ kvDataVersion = StringUtils.isNotBlank(currDataVersionString) ? JSON.parseObject(currDataVersionString, DataVersion.class) : new DataVersion();
+ return true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
Review Comment:
How about print some information for the exception here?
##########
broker/src/main/java/org/apache/rocketmq/broker/RocksDBConfigManager.java:
##########
@@ -14,46 +14,66 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.common.config;
+package org.apache.rocketmq.broker;
+import com.alibaba.fastjson.JSON;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.config.ConfigRocksDBStorage;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
+import org.apache.rocketmq.remoting.protocol.DataVersion;
import org.rocksdb.FlushOptions;
import org.rocksdb.RocksIterator;
import org.rocksdb.Statistics;
import org.rocksdb.WriteBatch;
+import java.nio.charset.StandardCharsets;
import java.util.function.BiConsumer;
public class RocksDBConfigManager {
protected static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-
- protected volatile boolean isStop = false;
- protected ConfigRocksDBStorage configRocksDBStorage = null;
+ public volatile boolean isStop = false;
+ public ConfigRocksDBStorage configRocksDBStorage = null;
private FlushOptions flushOptions = null;
private volatile long lastFlushMemTableMicroSecond = 0;
+
+ private final String filePath;
private final long memTableFlushInterval;
+ private DataVersion kvDataVersion = new DataVersion();
+
- public RocksDBConfigManager(long memTableFlushInterval) {
+ public RocksDBConfigManager(String filePath, long memTableFlushInterval) {
+ this.filePath = filePath;
this.memTableFlushInterval = memTableFlushInterval;
}
- public boolean load(String configFilePath, BiConsumer<byte[], byte[]> biConsumer) {
+ public boolean init() {
this.isStop = false;
- this.configRocksDBStorage = new ConfigRocksDBStorage(configFilePath);
- if (!this.configRocksDBStorage.start()) {
- return false;
- }
- RocksIterator iterator = this.configRocksDBStorage.iterator();
+ this.configRocksDBStorage = new ConfigRocksDBStorage(filePath);
+ return this.configRocksDBStorage.start();
+ }
+ public final boolean loadDataVersion() {
Review Comment:
Why use public final for method here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@rocketmq.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [ISSUE #8058]Support for upgrading metadata in json to rocksdb (#8045) [rocketmq]
Posted by "codecov-commenter (via GitHub)" <gi...@apache.org>.
codecov-commenter commented on PR #8116:
URL: https://github.com/apache/rocketmq/pull/8116#issuecomment-2105512083
## [Codecov](https://app.codecov.io/gh/apache/rocketmq/pull/8116?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) Report
Attention: Patch coverage is `54.71698%` with `96 lines` in your changes are missing coverage. Please review.
> Project coverage is 42.97%. Comparing base [(`159a603`)](https://app.codecov.io/gh/apache/rocketmq/commit/159a603219e363e5f991a0c85213f6dc13f0a9be?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) to head [(`2bf3840`)](https://app.codecov.io/gh/apache/rocketmq/pull/8116?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache).
| [Files](https://app.codecov.io/gh/apache/rocketmq/pull/8116?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) | Patch % | Lines |
|---|---|---|
| [...s/command/metadata/RocksDBConfigToJsonCommand.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=tools%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Ftools%2Fcommand%2Fmetadata%2FRocksDBConfigToJsonCommand.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-dG9vbHMvc3JjL21haW4vamF2YS9vcmcvYXBhY2hlL3JvY2tldG1xL3Rvb2xzL2NvbW1hbmQvbWV0YWRhdGEvUm9ja3NEQkNvbmZpZ1RvSnNvbkNvbW1hbmQuamF2YQ==) | 0.00% | [30 Missing :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [.../subscription/RocksDBSubscriptionGroupManager.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=broker%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fbroker%2Fsubscription%2FRocksDBSubscriptionGroupManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-YnJva2VyL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9yb2NrZXRtcS9icm9rZXIvc3Vic2NyaXB0aW9uL1JvY2tzREJTdWJzY3JpcHRpb25Hcm91cE1hbmFnZXIuamF2YQ==) | 67.60% | [17 Missing and 6 partials :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...e/rocketmq/common/config/ConfigRocksDBStorage.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=common%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fcommon%2Fconfig%2FConfigRocksDBStorage.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-Y29tbW9uL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9yb2NrZXRtcS9jb21tb24vY29uZmlnL0NvbmZpZ1JvY2tzREJTdG9yYWdlLmphdmE=) | 0.00% | [14 Missing :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...g/apache/rocketmq/broker/RocksDBConfigManager.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=broker%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fbroker%2FRocksDBConfigManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-YnJva2VyL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9yb2NrZXRtcS9icm9rZXIvUm9ja3NEQkNvbmZpZ01hbmFnZXIuamF2YQ==) | 72.22% | [5 Missing and 5 partials :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...cketmq/broker/topic/RocksDBTopicConfigManager.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=broker%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fbroker%2Ftopic%2FRocksDBTopicConfigManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-YnJva2VyL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9yb2NrZXRtcS9icm9rZXIvdG9waWMvUm9ja3NEQlRvcGljQ29uZmlnTWFuYWdlci5qYXZh) | 75.75% | [4 Missing and 4 partials :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...ache/rocketmq/broker/topic/TopicConfigManager.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=broker%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fbroker%2Ftopic%2FTopicConfigManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-YnJva2VyL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9yb2NrZXRtcS9icm9rZXIvdG9waWMvVG9waWNDb25maWdNYW5hZ2VyLmphdmE=) | 58.33% | [5 Missing :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [...ache/rocketmq/store/config/MessageStoreConfig.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=store%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fstore%2Fconfig%2FMessageStoreConfig.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-c3RvcmUvc3JjL21haW4vamF2YS9vcmcvYXBhY2hlL3JvY2tldG1xL3N0b3JlL2NvbmZpZy9NZXNzYWdlU3RvcmVDb25maWcuamF2YQ==) | 25.00% | [3 Missing :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [.../broker/subscription/SubscriptionGroupManager.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=broker%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Fbroker%2Fsubscription%2FSubscriptionGroupManager.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-YnJva2VyL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9yb2NrZXRtcS9icm9rZXIvc3Vic2NyaXB0aW9uL1N1YnNjcmlwdGlvbkdyb3VwTWFuYWdlci5qYXZh) | 80.00% | [2 Missing :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
| [.../apache/rocketmq/tools/command/MQAdminStartup.java](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&filepath=tools%2Fsrc%2Fmain%2Fjava%2Forg%2Fapache%2Frocketmq%2Ftools%2Fcommand%2FMQAdminStartup.java&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache#diff-dG9vbHMvc3JjL21haW4vamF2YS9vcmcvYXBhY2hlL3JvY2tldG1xL3Rvb2xzL2NvbW1hbmQvTVFBZG1pblN0YXJ0dXAuamF2YQ==) | 0.00% | [1 Missing :warning: ](https://app.codecov.io/gh/apache/rocketmq/pull/8116?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache) |
<details><summary>Additional details and impacted files</summary>
```diff
@@ Coverage Diff @@
## develop #8116 +/- ##
=============================================
+ Coverage 42.91% 42.97% +0.05%
- Complexity 10382 10428 +46
=============================================
Files 1270 1270
Lines 88694 88830 +136
Branches 11401 11402 +1
=============================================
+ Hits 38066 38177 +111
Misses 45935 45935
- Partials 4693 4718 +25
```
</details>
[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/apache/rocketmq/pull/8116?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache).
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=apache).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@rocketmq.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [ISSUE #8058]Support for upgrading metadata in json to rocksdb (#8045) [rocketmq]
Posted by "LetLetMe (via GitHub)" <gi...@apache.org>.
LetLetMe commented on PR #8116:
URL: https://github.com/apache/rocketmq/pull/8116#issuecomment-2123820289
> I checked the code, and it seems that upgrading from the 5.2.0 version with RocksDB enabled to this modified version is fully compatible, isn't it?
没问题
No problem
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@rocketmq.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [ISSUE #8058] Support for upgrading metadata in json to rocksdb (#8045) [rocketmq]
Posted by "LetLetMe (via GitHub)" <gi...@apache.org>.
LetLetMe closed pull request #8116: [ISSUE #8058] Support for upgrading metadata in json to rocksdb (#8045)
URL: https://github.com/apache/rocketmq/pull/8116
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@rocketmq.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org