You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by cl...@apache.org on 2016/07/06 14:10:00 UTC
[1/2] activemq-artemis git commit: More changes on the reclaimer for
better performance
Repository: activemq-artemis
Updated Branches:
refs/heads/master 98fa433bd -> 2b62bc74f
More changes on the reclaimer for better performance
Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/b8d6a374
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/b8d6a374
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/b8d6a374
Branch: refs/heads/master
Commit: b8d6a374a0303f69134c50a3c02ef52dc7a15c2a
Parents: 98fa433
Author: barreiro <lb...@gmail.com>
Authored: Wed May 25 03:23:28 2016 +0100
Committer: Clebert Suconic <cl...@apache.org>
Committed: Tue Jul 5 23:28:00 2016 -0400
----------------------------------------------------------------------
.../artemis/core/journal/impl/JournalFile.java | 16 +-
.../core/journal/impl/JournalFileImpl.java | 47 +++---
.../artemis/core/journal/impl/Reclaimer.java | 64 +++++---
.../unit/core/journal/impl/ReclaimerTest.java | 157 +++----------------
4 files changed, 96 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b8d6a374/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFile.java
----------------------------------------------------------------------
diff --git a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFile.java b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFile.java
index 83b3cf8..ade4da8 100644
--- a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFile.java
+++ b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFile.java
@@ -42,12 +42,18 @@ public interface JournalFile {
int getTotalNegativeToOthers();
/**
- * Whether this file's contents can deleted and the file reused.
- *
- * @param canDelete if {@code true} then this file's contents are unimportant and may be deleted
- * at any time.
+ * Whether this file additions all have a delete in some other file
*/
- void setCanReclaim(boolean canDelete);
+ boolean isPosReclaimCriteria();
+
+ void setPosReclaimCriteria();
+
+ /**
+ * Whether this file deletes are on files that are either marked for reclaim or have already been reclaimed
+ */
+ boolean isNegReclaimCriteria();
+
+ void setNegReclaimCriteria();
/**
* Whether this file's contents can deleted and the file reused.
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b8d6a374/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFileImpl.java
----------------------------------------------------------------------
diff --git a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFileImpl.java b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFileImpl.java
index 01d8000..210fcd6 100644
--- a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFileImpl.java
+++ b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/JournalFileImpl.java
@@ -16,9 +16,9 @@
*/
package org.apache.activemq.artemis.core.journal.impl;
-import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.activemq.artemis.core.io.SequentialFile;
@@ -37,13 +37,15 @@ public class JournalFileImpl implements JournalFile {
private final AtomicInteger liveBytes = new AtomicInteger(0);
- private boolean canReclaim;
+ // Flags to be used by determine if the journal file can be reclaimed
+ private boolean posReclaimCriteria = false;
+ private boolean negReclaimCriteria = false;
private final AtomicInteger totalNegativeToOthers = new AtomicInteger(0);
private final int version;
- private final Map<JournalFile, AtomicInteger> negCounts = new ConcurrentHashMap<>();
+ private final ConcurrentMap<JournalFile, AtomicInteger> negCounts = new ConcurrentHashMap<>();
public JournalFileImpl(final SequentialFile file, final long fileID, final int version) {
this.file = file;
@@ -61,13 +63,28 @@ public class JournalFileImpl implements JournalFile {
}
@Override
- public boolean isCanReclaim() {
- return canReclaim;
+ public boolean isPosReclaimCriteria() {
+ return posReclaimCriteria;
+ }
+
+ @Override
+ public void setPosReclaimCriteria() {
+ this.posReclaimCriteria = true;
+ }
+
+ @Override
+ public boolean isNegReclaimCriteria() {
+ return negReclaimCriteria;
}
@Override
- public void setCanReclaim(final boolean canReclaim) {
- this.canReclaim = canReclaim;
+ public void setNegReclaimCriteria() {
+ this.negReclaimCriteria = true;
+ }
+
+ @Override
+ public boolean isCanReclaim() {
+ return posReclaimCriteria && negReclaimCriteria;
}
@Override
@@ -75,7 +92,10 @@ public class JournalFileImpl implements JournalFile {
if (file != this) {
totalNegativeToOthers.incrementAndGet();
}
- getOrCreateNegCount(file).incrementAndGet();
+ AtomicInteger previous = negCounts.putIfAbsent(file, new AtomicInteger(1));
+ if (previous != null) {
+ previous.incrementAndGet();
+ }
}
@Override
@@ -152,17 +172,6 @@ public class JournalFileImpl implements JournalFile {
return builder.toString();
}
- private synchronized AtomicInteger getOrCreateNegCount(final JournalFile file) {
- AtomicInteger count = negCounts.get(file);
-
- if (count == null) {
- count = new AtomicInteger();
- negCounts.put(file, count);
- }
-
- return count;
- }
-
@Override
public void addSize(final int bytes) {
liveBytes.addAndGet(bytes);
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b8d6a374/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/Reclaimer.java
----------------------------------------------------------------------
diff --git a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/Reclaimer.java b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/Reclaimer.java
index c83e2af..accbe80 100644
--- a/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/Reclaimer.java
+++ b/artemis-journal/src/main/java/org/apache/activemq/artemis/core/journal/impl/Reclaimer.java
@@ -36,43 +36,55 @@ public class Reclaimer {
private static final Logger logger = Logger.getLogger(Reclaimer.class);
+ // The files are scanned in two stages. First we only check for 2) and do so while that criteria is not met.
+ // When 2) is met, set the first reclaim flag in the journal. After that point only check for 1)
+ // until that criteria is met as well. When 1) is met we set the second flag and the file can be reclaimed.
+
public void scan(final JournalFile[] files) {
for (int i = 0; i < files.length; i++) {
JournalFile currentFile = files[i];
- currentFile.setCanReclaim(true);
- // First we evaluate criterion 2)
- for (int j = i - 1; j >= 0; j--) {
- JournalFile file = files[j];
- if (currentFile.getNegCount(file) != 0 && !file.isCanReclaim()) {
- logger.tracef("%s can't be reclaimed because %s has negative values", currentFile, file);
- currentFile.setCanReclaim(false);
- break;
+ // criterion 2) --- this file deletes are from pos on files marked for reclaim or reclaimed
+ if (!currentFile.isNegReclaimCriteria()) {
+ boolean outstandingNeg = false;
+
+ for (int j = i - 1; j >= 0 && !outstandingNeg; j--) {
+ JournalFile file = files[j];
+ if (!file.isCanReclaim() && currentFile.getNegCount(file) != 0) {
+ logger.tracef("%s can't be reclaimed because %s has negative values", currentFile, file);
+ outstandingNeg = true;
+ }
+ }
+
+ if (outstandingNeg) {
+ continue; // Move to next file as we already know that this file can't be reclaimed because criterion 2)
+ }
+ else {
+ currentFile.setNegReclaimCriteria();
}
- }
- if (!currentFile.isCanReclaim()) {
- continue; // Move to next file as we already know that this file can't be reclaimed because criterion 2)
}
- // Now we evaluate criterion 1)
- int negCount = 0, posCount = currentFile.getPosCount();
- logger.tracef("posCount on %s = %d", currentFile, posCount);
+ // criterion 1) --- this files additions all have matching deletes
+ if (!currentFile.isPosReclaimCriteria()) {
+ int negCount = 0, posCount = currentFile.getPosCount();
+ logger.tracef("posCount on %s = %d", currentFile, posCount);
- for (int j = i; j < files.length && negCount < posCount; j++) {
- int toNeg = files[j].getNegCount(currentFile);
- negCount += toNeg;
+ for (int j = i; j < files.length && negCount < posCount; j++) {
+ int toNeg = files[j].getNegCount(currentFile);
+ negCount += toNeg;
- if (logger.isTraceEnabled() && toNeg != 0) {
- logger.tracef("Negative from %s into %s = %d", files[j], currentFile, toNeg);
+ if (logger.isTraceEnabled() && toNeg != 0) {
+ logger.tracef("Negative from %s into %s = %d", files[j], currentFile, toNeg);
+ }
}
- }
- if (negCount < posCount ) {
- logger.tracef("%s can't be reclaimed because there are not enough negatives %d", currentFile, negCount);
- currentFile.setCanReclaim(false);
- }
- else {
- logger.tracef("%s can be reclaimed", currentFile);
+ if (negCount < posCount ) {
+ logger.tracef("%s can't be reclaimed because there are not enough negatives %d", currentFile, negCount);
+ }
+ else {
+ logger.tracef("%s can be reclaimed", currentFile);
+ currentFile.setPosReclaimCriteria();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/activemq-artemis/blob/b8d6a374/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/journal/impl/ReclaimerTest.java
----------------------------------------------------------------------
diff --git a/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/journal/impl/ReclaimerTest.java b/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/journal/impl/ReclaimerTest.java
index 39132a5..8216800 100644
--- a/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/journal/impl/ReclaimerTest.java
+++ b/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/journal/impl/ReclaimerTest.java
@@ -16,22 +16,16 @@
*/
package org.apache.activemq.artemis.tests.unit.core.journal.impl;
-import org.apache.activemq.artemis.tests.util.ActiveMQTestBase;
-import org.junit.Before;
-
-import org.junit.Test;
-
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
-import java.util.Set;
-
-import org.junit.Assert;
-
import org.apache.activemq.artemis.core.io.SequentialFile;
import org.apache.activemq.artemis.core.journal.impl.JournalFile;
import org.apache.activemq.artemis.core.journal.impl.JournalImpl;
import org.apache.activemq.artemis.core.journal.impl.Reclaimer;
+import org.apache.activemq.artemis.tests.util.ActiveMQTestBase;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
public class ReclaimerTest extends ActiveMQTestBase {
@@ -722,42 +716,25 @@ public class ReclaimerTest extends ActiveMQTestBase {
static final class MockJournalFile implements JournalFile {
- private final Set<Long> transactionIDs = new HashSet<>();
-
- private final Set<Long> transactionTerminationIDs = new HashSet<>();
-
- private final Set<Long> transactionPrepareIDs = new HashSet<>();
-
private final Map<JournalFile, Integer> negCounts = new HashMap<>();
private int posCount;
- private boolean canDelete;
-
- private boolean needCleanup;
-
private int totalDep;
- public void extendOffset(final int delta) {
- }
+ private boolean posReclaimCriteria;
+ private boolean negReclaimCriteria;
@Override
public SequentialFile getFile() {
return null;
}
- public long getOffset() {
- return 0;
- }
-
@Override
public long getFileID() {
return 0;
}
- public void setOffset(final long offset) {
- }
-
@Override
public int getNegCount(final JournalFile file) {
Integer count = negCounts.get(file);
@@ -797,75 +774,28 @@ public class ReclaimerTest extends ActiveMQTestBase {
}
@Override
- public boolean isCanReclaim() {
- return canDelete;
+ public boolean isPosReclaimCriteria() {
+ return posReclaimCriteria;
}
@Override
- public void setCanReclaim(final boolean canDelete) {
- this.canDelete = canDelete;
- }
-
- public void addTransactionID(final long id) {
- transactionIDs.add(id);
- }
-
- public void addTransactionPrepareID(final long id) {
- transactionPrepareIDs.add(id);
- }
-
- public void addTransactionTerminationID(final long id) {
- transactionTerminationIDs.add(id);
- }
-
- public boolean containsTransactionID(final long id) {
- return transactionIDs.contains(id);
- }
-
- public boolean containsTransactionPrepareID(final long id) {
- return transactionPrepareIDs.contains(id);
- }
-
- public boolean containsTransactionTerminationID(final long id) {
- return transactionTerminationIDs.contains(id);
- }
-
- public Set<Long> getTranactionTerminationIDs() {
- return transactionTerminationIDs;
- }
-
- public Set<Long> getTransactionPrepareIDs() {
- return transactionPrepareIDs;
- }
-
- public Set<Long> getTransactionsIDs() {
- return transactionIDs;
- }
-
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#decPendingTransaction()
- */
- public void decPendingTransaction() {
+ public void setPosReclaimCriteria() {
+ this.posReclaimCriteria = true;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getPendingTransactions()
- */
- public int getPendingTransactions() {
- return 0;
+ @Override
+ public boolean isNegReclaimCriteria() {
+ return negReclaimCriteria;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#incPendingTransaction()
- */
- public void incPendingTransaction() {
+ @Override
+ public void setNegReclaimCriteria() {
+ this.negReclaimCriteria = true;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getOrderingID()
- */
- public int getOrderingID() {
- return 0;
+ @Override
+ public boolean isCanReclaim() {
+ return posReclaimCriteria && negReclaimCriteria;
}
@Override
@@ -876,74 +806,25 @@ public class ReclaimerTest extends ActiveMQTestBase {
public void decSize(final int bytes) {
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getSize()
- */
@Override
public int getLiveSize() {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#isNeedCleanup()
- */
- public boolean isNeedCleanup() {
- return needCleanup;
- }
-
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#resetNegCount(org.apache.activemq.artemis.core.journal.impl.JournalFile)
- */
- public boolean resetNegCount(final JournalFile file) {
- return false;
- }
-
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#setNeedCleanup(boolean)
- */
- public void setNeedCleanup(final boolean needCleanup) {
- this.needCleanup = needCleanup;
-
- }
-
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getRecordID()
- */
@Override
public int getRecordID() {
return 0;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getTotalNegativeToOthers()
- */
@Override
public int getTotalNegativeToOthers() {
return totalDep;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getJournalVersion()
- */
@Override
public int getJournalVersion() {
return JournalImpl.FORMAT_VERSION;
}
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#getTotNeg()
- */
- public int getTotNeg() {
- // TODO Auto-generated method stub
- return 0;
- }
-
- /* (non-Javadoc)
- * @see org.apache.activemq.artemis.core.journal.impl.JournalFile#setTotNeg(int)
- */
- public void setTotNeg(int totNeg) {
- // TODO Auto-generated method stub
-
- }
}
}
[2/2] activemq-artemis git commit: This closes #549
Posted by cl...@apache.org.
This closes #549
Project: http://git-wip-us.apache.org/repos/asf/activemq-artemis/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq-artemis/commit/2b62bc74
Tree: http://git-wip-us.apache.org/repos/asf/activemq-artemis/tree/2b62bc74
Diff: http://git-wip-us.apache.org/repos/asf/activemq-artemis/diff/2b62bc74
Branch: refs/heads/master
Commit: 2b62bc74fa66926acd634eb55e6c111a0dba0c50
Parents: 98fa433 b8d6a37
Author: Clebert Suconic <cl...@apache.org>
Authored: Wed Jul 6 10:09:48 2016 -0400
Committer: Clebert Suconic <cl...@apache.org>
Committed: Wed Jul 6 10:09:48 2016 -0400
----------------------------------------------------------------------
.../artemis/core/journal/impl/JournalFile.java | 16 +-
.../core/journal/impl/JournalFileImpl.java | 47 +++---
.../artemis/core/journal/impl/Reclaimer.java | 64 +++++---
.../unit/core/journal/impl/ReclaimerTest.java | 157 +++----------------
4 files changed, 96 insertions(+), 188 deletions(-)
----------------------------------------------------------------------