You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@rocketmq.apache.org by GitBox <gi...@apache.org> on 2019/02/21 06:16:02 UTC
[GitHub] ShannonDing commented on a change in pull request #836: [RIP-10]
Add test cases for DefaultMessageStore.CleanCommitLogService and
DefaultMessageStore.CleanConsumeQueueService
ShannonDing commented on a change in pull request #836: [RIP-10] Add test cases for DefaultMessageStore.CleanCommitLogService and DefaultMessageStore.CleanConsumeQueueService
URL: https://github.com/apache/rocketmq/pull/836#discussion_r258791639
##########
File path: store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreCleanFilesTest.java
##########
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.store;
+
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.store.config.FlushDiskType;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.Calendar;
+import java.util.Map;
+
+import static org.apache.rocketmq.common.message.MessageDecoder.CHARSET_UTF8;
+import static org.apache.rocketmq.store.ConsumeQueue.CQ_STORE_UNIT_SIZE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class DefaultMessageStoreCleanFilesTest {
+ private DefaultMessageStore messageStore;
+ private SocketAddress bornHost;
+ private SocketAddress storeHost;
+
+ private String topic = "test";
+ private int queueId = 0;
+ private int fileCountCommitLog = 55;
+ // exactly one message per CommitLog file.
+ private int msgCount = fileCountCommitLog;
+ private int mappedFileSize = 128;
+ private int fileReservedTime = 1;
+
+ @Before
+ public void init() throws Exception {
+ storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);
+ bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0);
+ }
+
+ /**
+ * make sure disk space usage is less than 85%.
+ * if disk space usage is greater than 85%, by default, it will trigger the deletion at once.
+ */
+ @Test
+ public void testDeleteExpiredFilesByTimeUp() throws Exception {
+ String deleteWhen = Calendar.getInstance().get(Calendar.HOUR_OF_DAY) + "";
+ int diskMaxUsedSpaceRatio = 95; // the max value of diskMaxUsedSpaceRatio
+ messageStore = initMessageStore(deleteWhen, diskMaxUsedSpaceRatio);
+
+ // build and put 55 messages, exactly one message per CommitLog file.
+ buildAndPutMessagesToMessageStore(msgCount);
+
+ // undo comment out the code below, if want to debug this case rather than just run it.
+ // Thread.sleep(1000 * 60 + 100);
+
+ MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();
+ assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());
+
+ int fileCountConsumeQueue = getFileCountConsumeQueue();
+ MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();
+ assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());
+
+ int expireFileCount = 15;
+ expireFiles(commitLogQueue, expireFileCount);
+
+ // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX
+ for (int a = 1, fileCount = expireFileCount; a <= (int) Math.ceil((double) expireFileCount / 10); a++, fileCount -= 10) {
+ getCleanCommitLogService().run();
+ getCleanConsumeQueueService().run();
+
+ int expectDeletedCount = fileCount >= 10 ? a * 10 : ((a - 1) * 10 + fileCount);
+ assertEquals(fileCountCommitLog - expectDeletedCount, commitLogQueue.getMappedFiles().size());
+
+ int msgCountPerFile = getMsgCountPerConsumeQueueMappedFile();
+ int expectDeleteCountConsumeQueue = (int) Math.floor((double) expectDeletedCount / msgCountPerFile);
+ assertEquals(fileCountConsumeQueue - expectDeleteCountConsumeQueue, consumeQueue.getMappedFiles().size());
+ }
+ }
+
+ /**
+ * make sure disk space usage is greater than 10%, less than 85%.
+ * if disk space usage is greater than 85%, by default, it will trigger the deletion at once.
+ *
+ * @see DefaultMessageStoreCleanFilesTest#testDeleteFilesImmediatelyBySpaceFull()
+ */
+ @Test
+ public void testDeleteExpiredFilesBySpaceFull() throws Exception {
+ String deleteWhen = "04";
+ // the min value of diskMaxUsedSpaceRatio. make sure disk space usage is greater than 10%
+ int diskMaxUsedSpaceRatio = 10;
+ messageStore = initMessageStore(deleteWhen, diskMaxUsedSpaceRatio);
+
+ // build and put 55 messages, exactly one message per CommitLog file.
+ buildAndPutMessagesToMessageStore(msgCount);
+
+ // undo comment out the code below, if want to debug this case rather than just run it.
+ // Thread.sleep(1000 * 60 + 100);
+
+ MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();
+ assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());
+
+ int fileCountConsumeQueue = getFileCountConsumeQueue();
+ MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();
+ assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());
+
+ int expireFileCount = 15;
+ expireFiles(commitLogQueue, expireFileCount);
+
+ // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX
+ for (int a = 1, fileCount = expireFileCount; a <= (int) Math.ceil((double) expireFileCount / 10); a++, fileCount -= 10) {
+ getCleanCommitLogService().run();
+ getCleanConsumeQueueService().run();
+
+ int expectDeletedCount = fileCount >= 10 ? a * 10 : ((a - 1) * 10 + fileCount);
+ assertEquals(fileCountCommitLog - expectDeletedCount, commitLogQueue.getMappedFiles().size());
+
+ int msgCountPerFile = getMsgCountPerConsumeQueueMappedFile();
+ int expectDeleteCountConsumeQueue = (int) Math.floor((double) expectDeletedCount / msgCountPerFile);
+ assertEquals(fileCountConsumeQueue - expectDeleteCountConsumeQueue, consumeQueue.getMappedFiles().size());
+ }
+ }
+
+
+ /**
+ * run with the vm args:
+ * -Drocketmq.broker.diskSpaceWarningLevelRatio=0.1
+ * -Drocketmq.broker.diskSpaceCleanForciblyRatio=0.1
+ * <p>
+ * make sure disk space usage is greater than 10%
+ */
+ @Test
+ public void testDeleteFilesImmediatelyBySpaceFull() throws Exception {
+ String deleteWhen = "04";
+ // the min value of diskMaxUsedSpaceRatio. make sure disk space usage is greater than 10%
+ int diskMaxUsedSpaceRatio = 10;
+
+ messageStore = initMessageStore(deleteWhen, diskMaxUsedSpaceRatio);
+
+ // build and put 55 messages, exactly one message per CommitLog file.
+ buildAndPutMessagesToMessageStore(msgCount);
+
+ // undo comment out the code below, if want to debug this case rather than just run it.
+ // Thread.sleep(1000 * 60 + 100);
+
+ MappedFileQueue commitLogQueue = getMappedFileQueueCommitLog();
+ assertEquals(fileCountCommitLog, commitLogQueue.getMappedFiles().size());
+
+ int fileCountConsumeQueue = getFileCountConsumeQueue();
+ MappedFileQueue consumeQueue = getMappedFileQueueConsumeQueue();
+ assertEquals(fileCountConsumeQueue, consumeQueue.getMappedFiles().size());
+
+ // In this case, there is no need to expire the files.
+ // int expireFileCount = 15;
+ // expireFiles(commitLogQueue, expireFileCount);
+
+ // magic code 10 reference to MappedFileQueue#DELETE_FILES_BATCH_MAX
+ for (int a = 1, fileCount = fileCountCommitLog;
+ a <= (int) Math.ceil((double) fileCountCommitLog / 10) && fileCount >= 10;
+ a++, fileCount -= 10) {
+ getCleanCommitLogService().run();
+ getCleanConsumeQueueService().run();
+
+ assertEquals(fileCountCommitLog - 10 * a, commitLogQueue.getMappedFiles().size());
Review comment:
Could you please check this test case again? you can refer to the CI result like this:
```
org.apache.rocketmq.store.DefaultMessageStoreCleanFilesTest
testDeleteFilesImmediatelyBySpaceFull(org.apache.rocketmq.store.DefaultMessageStoreCleanFilesTest) Time elapsed: 1.355 sec <<< FAILURE!
java.lang.AssertionError: expected:<45> but was:<55>
at org.apache.rocketmq.store.DefaultMessageStoreCleanFilesTest.testDeleteFilesImmediatelyBySpaceFull(DefaultMessageStoreCleanFilesTest.java:184)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services