You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by ta...@apache.org on 2016/03/03 23:35:30 UTC
[1/2] activemq git commit:
https://issues.apache.org/jira/browse/AMQ-5578
Repository: activemq
Updated Branches:
refs/heads/master be4ad3d1c -> 5e0f49332
https://issues.apache.org/jira/browse/AMQ-5578
Add additional preallocation strategy [CHUNKED contributed by user
jtahlborn.
Project: http://git-wip-us.apache.org/repos/asf/activemq/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq/commit/edfc23ee
Tree: http://git-wip-us.apache.org/repos/asf/activemq/tree/edfc23ee
Diff: http://git-wip-us.apache.org/repos/asf/activemq/diff/edfc23ee
Branch: refs/heads/master
Commit: edfc23ee9d019380aca287e02bd712a2bb557993
Parents: be4ad3d
Author: Timothy Bish <ta...@gmail.com>
Authored: Thu Mar 3 17:34:07 2016 -0500
Committer: Timothy Bish <ta...@gmail.com>
Committed: Thu Mar 3 17:34:07 2016 -0500
----------------------------------------------------------------------
.../store/kahadb/disk/journal/Journal.java | 37 +++++++++++++++++---
1 file changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/activemq/blob/edfc23ee/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
----------------------------------------------------------------------
diff --git a/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java b/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
index f696ea4..ed2cba3 100644
--- a/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
+++ b/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
@@ -62,6 +62,8 @@ public class Journal {
private static final int MAX_BATCH_SIZE = 32*1024*1024;
+ private static final int PREALLOC_CHUNK_SIZE = 1 << 20;
+
// ITEM_HEAD_SPACE = length + type+ reserved space + SOR
public static final int RECORD_HEAD_SPACE = 4 + 1;
@@ -69,10 +71,10 @@ public class Journal {
public static final byte BATCH_CONTROL_RECORD_TYPE = 2;
// Batch Control Item holds a 4 byte size of the batch and a 8 byte checksum of the batch.
public static final byte[] BATCH_CONTROL_RECORD_MAGIC = bytes("WRITE BATCH");
- public static final int BATCH_CONTROL_RECORD_SIZE = RECORD_HEAD_SPACE+BATCH_CONTROL_RECORD_MAGIC.length+4+8;
+ public static final int BATCH_CONTROL_RECORD_SIZE = RECORD_HEAD_SPACE + BATCH_CONTROL_RECORD_MAGIC.length + 4 + 8;
public static final byte[] BATCH_CONTROL_RECORD_HEADER = createBatchControlRecordHeader();
- // tackle corruption when checksum is disabled or corrupt with zeros, minimise data loss
+ // tackle corruption when checksum is disabled or corrupt with zeros, minimize data loss
public void corruptRecoveryLocation(Location recoveryPosition) throws IOException {
DataFile dataFile = getDataFile(recoveryPosition);
// with corruption on recovery we have no faith in the content - slip to the next batch record or eof
@@ -87,7 +89,6 @@ public class Journal {
recoveryPosition.setSize(-1);
dataFile.corruptedBlocks.add(sequence);
-
} catch (IOException e) {
} finally {
accessorPool.closeDataFileAccessor(reader);
@@ -97,7 +98,8 @@ public class Journal {
public enum PreallocationStrategy {
SPARSE_FILE,
OS_KERNEL_COPY,
- ZEROS;
+ ZEROS,
+ CHUNKED_ZEROS;
}
public enum PreallocationScope {
@@ -256,6 +258,8 @@ public class Journal {
doPreallocationKernelCopy(file);
} else if (PreallocationStrategy.ZEROS == preallocationStrategy) {
doPreallocationZeros(file);
+ } else if (PreallocationStrategy.CHUNKED_ZEROS == preallocationStrategy) {
+ doPreallocationChunkedZeros(file);
} else {
doPreallocationSparseFile(file);
}
@@ -321,6 +325,31 @@ public class Journal {
return rc;
}
+ private void doPreallocationChunkedZeros(RecoverableRandomAccessFile file) {
+
+ ByteBuffer buffer = ByteBuffer.allocate(PREALLOC_CHUNK_SIZE);
+ buffer.position(0);
+ buffer.limit(PREALLOC_CHUNK_SIZE);
+
+ try {
+ FileChannel channel = file.getChannel();
+
+ int remLen = maxFileLength;
+ while (remLen > 0) {
+ if (remLen < buffer.remaining()) {
+ buffer.limit(remLen);
+ }
+ int writeLen = channel.write(buffer);
+ remLen -= writeLen;
+ buffer.rewind();
+ }
+
+ channel.force(false);
+ channel.position(0);
+ } catch (IOException e) {
+ LOG.error("Could not preallocate journal file with zeros! Will continue without preallocation", e);
+ }
+ }
private static byte[] bytes(String string) {
try {
return string.getBytes("UTF-8");
[2/2] activemq git commit:
https://issues.apache.org/jira/browse/AMQ-5578
Posted by ta...@apache.org.
https://issues.apache.org/jira/browse/AMQ-5578
Remove unnecessary buffer initialization after confirmation that it is
not needed.
Project: http://git-wip-us.apache.org/repos/asf/activemq/repo
Commit: http://git-wip-us.apache.org/repos/asf/activemq/commit/5e0f4933
Tree: http://git-wip-us.apache.org/repos/asf/activemq/tree/5e0f4933
Diff: http://git-wip-us.apache.org/repos/asf/activemq/diff/5e0f4933
Branch: refs/heads/master
Commit: 5e0f49332be2a0b30cb40dc9e735a794fe7b78f6
Parents: edfc23e
Author: Timothy Bish <ta...@gmail.com>
Authored: Thu Mar 3 17:35:21 2016 -0500
Committer: Timothy Bish <ta...@gmail.com>
Committed: Thu Mar 3 17:35:21 2016 -0500
----------------------------------------------------------------------
.../apache/activemq/store/kahadb/disk/journal/Journal.java | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/activemq/blob/5e0f4933/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
----------------------------------------------------------------------
diff --git a/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java b/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
index ed2cba3..a89f2a1 100644
--- a/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
+++ b/activemq-kahadb-store/src/main/java/org/apache/activemq/store/kahadb/disk/journal/Journal.java
@@ -279,12 +279,7 @@ public class Journal {
private void doPreallocationZeros(RecoverableRandomAccessFile file) {
ByteBuffer buffer = ByteBuffer.allocate(maxFileLength);
- // intentional double initialization due to interaction with the OS level
- // file allocation mechanics.
- for (int i = 0; i < maxFileLength; i++) {
- buffer.put((byte) 0x00);
- }
- buffer.flip();
+ buffer.limit(maxFileLength);
try {
FileChannel channel = file.getChannel();