You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/06/19 08:11:32 UTC
[04/11] james-project git commit: JAMES-2426 Store size metadata in
ZIP
JAMES-2426 Store size metadata in ZIP
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/feb18013
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/feb18013
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/feb18013
Branch: refs/heads/master
Commit: feb18013f50d9fad042f89f2dae63725b9372e41
Parents: 84338dc
Author: Raphael Ouazana <ra...@linagora.com>
Authored: Thu Jun 14 17:29:15 2018 +0200
Committer: benwa <bt...@linagora.com>
Committed: Tue Jun 19 15:07:55 2018 +0700
----------------------------------------------------------------------
.../james/mailbox/backup/SizeExtraField.java | 110 ++++++++++
.../org/apache/james/mailbox/backup/Zipper.java | 9 +-
.../mailbox/backup/MailboxMessageFixture.java | 4 +-
.../mailbox/backup/SizeExtraFieldTest.java | 202 +++++++++++++++++++
.../mailbox/backup/ZipArchiveEntryAssert.java | 25 +++
.../apache/james/mailbox/backup/ZipperTest.java | 13 ++
6 files changed, 359 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java
----------------------------------------------------------------------
diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java
new file mode 100644
index 0000000..4da7f9c
--- /dev/null
+++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/SizeExtraField.java
@@ -0,0 +1,110 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mailbox.backup;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.zip.ZipException;
+
+import org.apache.commons.compress.archivers.zip.ZipExtraField;
+import org.apache.commons.compress.archivers.zip.ZipShort;
+
+public class SizeExtraField implements ZipExtraField {
+ public static final ZipShort ID = new ZipShort(0x6A61); // "aj" in little-endian
+
+ private Optional<Long> size;
+
+ public SizeExtraField() {
+ this(Optional.empty());
+ }
+
+ public SizeExtraField(long size) {
+ this(Optional.of(size));
+ }
+
+ public SizeExtraField(Optional<Long> size) {
+ this.size = size;
+ }
+
+ @Override
+ public ZipShort getHeaderId() {
+ return ID;
+ }
+
+ @Override
+ public ZipShort getLocalFileDataLength() {
+ return new ZipShort(Long.BYTES);
+ }
+
+ @Override
+ public ZipShort getCentralDirectoryLength() {
+ return getLocalFileDataLength();
+ }
+
+ @Override
+ public byte[] getLocalFileDataData() {
+ long value = size.orElseThrow(() -> new RuntimeException("Value must by initialized"));
+ return ByteBuffer.allocate(Long.BYTES)
+ .order(ByteOrder.LITTLE_ENDIAN)
+ .putLong(value)
+ .array();
+ }
+
+ @Override
+ public byte[] getCentralDirectoryData() {
+ return getLocalFileDataData();
+ }
+
+ @Override
+ public void parseFromLocalFileData(byte[] buffer, int offset, int length) throws ZipException {
+ if (length != Long.BYTES) {
+ throw new ZipException("Unexpected data length for SizeExtraField. Expected " + Long.BYTES + " but got " + length + ".");
+ }
+ size = Optional.of(ByteBuffer
+ .wrap(buffer, offset, Long.BYTES)
+ .order(ByteOrder.LITTLE_ENDIAN)
+ .getLong());
+ }
+
+ @Override
+ public void parseFromCentralDirectoryData(byte[] buffer, int offset, int length) throws ZipException {
+ parseFromLocalFileData(buffer, offset, length);
+ }
+
+ public Optional<Long> getSize() {
+ return size;
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (o instanceof SizeExtraField) {
+ SizeExtraField that = (SizeExtraField) o;
+
+ return Objects.equals(this.size, that.size);
+ }
+ return false;
+ }
+
+ @Override
+ public final int hashCode() {
+ return Objects.hash(size);
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java
----------------------------------------------------------------------
diff --git a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java
index c6d95ad..e4c2518 100644
--- a/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java
+++ b/mailbox/backup/src/main/java/org/apache/james/mailbox/backup/Zipper.java
@@ -23,12 +23,16 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
-import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ExtraFieldUtils;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.store.mail.model.MailboxMessage;
public class Zipper implements Backup {
+ public Zipper() {
+ ExtraFieldUtils.register(SizeExtraField.class);
+ }
@Override
public void archive(List<MailboxMessage> messages, OutputStream destination) throws IOException {
@@ -42,7 +46,8 @@ public class Zipper implements Backup {
private void storeInArchive(MailboxMessage message, ZipArchiveOutputStream archiveOutputStream) throws IOException {
String entryId = message.getMessageId().serialize();
- ArchiveEntry archiveEntry = archiveOutputStream.createArchiveEntry(new File(entryId), entryId);
+ ZipArchiveEntry archiveEntry = (ZipArchiveEntry) archiveOutputStream.createArchiveEntry(new File(entryId), entryId);
+ archiveEntry.addExtraField(new SizeExtraField(message.getFullContentOctets()));
archiveOutputStream.putArchiveEntry(archiveEntry);
IOUtils.copy(message.getFullContent(), archiveOutputStream);
archiveOutputStream.closeArchiveEntry();
http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java
----------------------------------------------------------------------
diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java
index 8012e78..3d46075 100644
--- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java
+++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/MailboxMessageFixture.java
@@ -49,8 +49,8 @@ public interface MailboxMessageFixture {
SharedByteArrayInputStream CONTENT_STREAM_2 = new SharedByteArrayInputStream(MESSAGE_CONTENT_2.getBytes(MESSAGE_CHARSET));
MessageId MESSAGE_ID_1 = MESSAGE_ID_FACTORY.generate();
MessageId MESSAGE_ID_2 = MESSAGE_ID_FACTORY.generate();
- int SIZE_1 = 1000;
- int SIZE_2 = 2000;
+ long SIZE_1 = 1000;
+ long SIZE_2 = 2000;
SimpleMailboxMessage MESSAGE_1 = SimpleMailboxMessage.builder()
.messageId(MESSAGE_ID_1)
http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java
----------------------------------------------------------------------
diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java
new file mode 100644
index 0000000..db2b274
--- /dev/null
+++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/SizeExtraFieldTest.java
@@ -0,0 +1,202 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+package org.apache.james.mailbox.backup;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.zip.ZipException;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.shaded.org.bouncycastle.util.Arrays;
+
+import com.google.common.base.Charsets;
+
+public class SizeExtraFieldTest {
+ private static final byte[] ZERO_AS_BYTE_ARRAY = {0, 0, 0, 0, 0, 0, 0, 0};
+ private static final byte[] _123456789ABCDEF0_AS_LE_BYTE_ARRAY = new byte[] {(byte) 0xF0, (byte) 0xDE, (byte) 0xBC, (byte) 0x9A, 0x78, 0x56, 0x34, 0x12};
+ private static final byte[] FEDCBA9876543210_AS_LE_BYTE_ARRAY = new byte[] {0x10, 0x32, 0x54, 0x76, (byte) 0x98, (byte) 0xBA, (byte) 0xDC, (byte) 0xFE};
+ private static final byte[] UNUSED = new byte[] {(byte) 0xDE, (byte) 0xAD};
+
+ private SizeExtraField testee;
+
+ @BeforeEach
+ void setUp() {
+ testee = new SizeExtraField();
+ }
+
+ @Test
+ void getLocalFileDataLengthShouldReturnIntegerSize() {
+ assertThat(testee.getLocalFileDataLength().getValue())
+ .isEqualTo(Long.BYTES);
+ }
+
+ @Test
+ void getCentralDirectoryLengthShouldReturnIntegerSize() {
+ assertThat(testee.getCentralDirectoryLength().getValue())
+ .isEqualTo(Long.BYTES);
+ }
+
+ @Test
+ void getHeaderIdShouldReturnSpecificStringInLittleEndian() {
+ ByteBuffer byteBuffer = ByteBuffer.wrap(testee.getHeaderId().getBytes())
+ .order(ByteOrder.LITTLE_ENDIAN);
+ assertThat(Charsets.US_ASCII.decode(byteBuffer).toString())
+ .isEqualTo("aj");
+ }
+
+ @Test
+ void getLocalFileDataDataShouldThrowWhenNoValue() {
+ assertThatThrownBy(() -> testee.getLocalFileDataData())
+ .isInstanceOf(RuntimeException.class);
+ }
+
+ @Test
+ void getLocalFileDataDataShouldReturnZeroWhenZero() {
+ byte[] actual = new SizeExtraField(0).getLocalFileDataData();
+ assertThat(actual).isEqualTo(ZERO_AS_BYTE_ARRAY);
+ }
+
+ @Test
+ void getLocalFileDataDataShouldReturnValueInLittleIndianWhen123456789ABCDEF0() {
+ byte[] actual = new SizeExtraField(0x123456789ABCDEF0L).getLocalFileDataData();
+ assertThat(actual).isEqualTo(_123456789ABCDEF0_AS_LE_BYTE_ARRAY);
+ }
+
+ @Test
+ void getLocalFileDataDataShouldReturnValueInLittleIndianWhenFEDCBA9876543210() {
+ byte[] actual = new SizeExtraField(0xFEDCBA9876543210L).getLocalFileDataData();
+ assertThat(actual).isEqualTo(FEDCBA9876543210_AS_LE_BYTE_ARRAY);
+ }
+
+ @Test
+ void getCentralDirectoryDataShouldThrowWhenNoValue() {
+ assertThatThrownBy(() -> testee.getCentralDirectoryData())
+ .isInstanceOf(RuntimeException.class);
+ }
+
+ @Test
+ void getCentralDirectoryDataShouldReturnZeroWhenZero() {
+ byte[] actual = new SizeExtraField(0).getCentralDirectoryData();
+ assertThat(actual).isEqualTo(ZERO_AS_BYTE_ARRAY);
+ }
+
+ @Test
+ void getCentralDirectoryDataShouldReturnValueInLittleIndianWhen123456789ABCDEF0() {
+ byte[] actual = new SizeExtraField(0x123456789ABCDEF0L).getCentralDirectoryData();
+ assertThat(actual).isEqualTo(_123456789ABCDEF0_AS_LE_BYTE_ARRAY);
+ }
+
+ @Test
+ void getCentralDirectoryDataShouldReturnValueInLittleIndianWhenFEDCBA9876543210() {
+ byte[] actual = new SizeExtraField(0xFEDCBA9876543210L).getCentralDirectoryData();
+ assertThat(actual).isEqualTo(FEDCBA9876543210_AS_LE_BYTE_ARRAY);
+ }
+
+ @Test
+ void parseFromLocalFileDataShouldThrownWhenLengthIsSmallerThan8() {
+ byte[] input = new byte[] {0, 0, 0, 0, 0, 0, 0};
+ assertThatThrownBy(() -> testee.parseFromLocalFileData(input, 0, 7))
+ .isInstanceOf(ZipException.class);
+ }
+
+ @Test
+ void parseFromLocalFileDataShouldThrownWhenLengthIsBiggerThan8() {
+ byte[] input = new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 0};
+ assertThatThrownBy(() -> testee.parseFromLocalFileData(input, 0, 9))
+ .isInstanceOf(ZipException.class);
+ }
+
+ @Test
+ void parseFromLocalFileDataShouldParseWhenZero() throws Exception {
+ testee.parseFromLocalFileData(ZERO_AS_BYTE_ARRAY, 0, 8);
+ assertThat(testee.getSize())
+ .contains(0L);
+ }
+
+ @Test
+ void parseFromLocalFileDataShouldParseWhen123456789ABCDEF0InLittleEndian() throws Exception {
+ testee.parseFromLocalFileData(_123456789ABCDEF0_AS_LE_BYTE_ARRAY, 0, 8);
+ assertThat(testee.getSize())
+ .contains(0x123456789ABCDEF0L);
+ }
+
+ @Test
+ void parseFromLocalFileDataShouldParseWhenFEDCBA9876543210InLittleEndian() throws Exception {
+ byte[] input = FEDCBA9876543210_AS_LE_BYTE_ARRAY;
+ testee.parseFromLocalFileData(input, 0, 8);
+ assertThat(testee.getSize())
+ .contains(0xFEDCBA9876543210L);
+ }
+
+ @Test
+ void parseFromLocalFileDataShouldHandleOffset() throws Exception {
+ byte[] input = Arrays.concatenate(UNUSED, _123456789ABCDEF0_AS_LE_BYTE_ARRAY);
+ testee.parseFromLocalFileData(input, 2, 8);
+ assertThat(testee.getSize())
+ .contains(0x123456789ABCDEF0L);
+ }
+
+ @Test
+ void parseFromCentralDirectoryDataShouldThrownWhenLengthIsSmallerThan8() {
+ byte[] input = new byte[7];
+ assertThatThrownBy(() -> testee.parseFromCentralDirectoryData(input, 0, 7))
+ .isInstanceOf(ZipException.class);
+ }
+
+ @Test
+ void parseFromCentralDirectoryDataShouldThrownWhenLengthIsBiggerThan8() {
+ byte[] input = new byte[9];
+ assertThatThrownBy(() -> testee.parseFromCentralDirectoryData(input, 0, 9))
+ .isInstanceOf(ZipException.class);
+ }
+
+ @Test
+ void parseFromCentralDirectoryDataShouldParseWhenZero() throws Exception {
+ testee.parseFromCentralDirectoryData(ZERO_AS_BYTE_ARRAY, 0, 8);
+ assertThat(testee.getSize())
+ .contains(0L);
+ }
+
+ @Test
+ void parseFromCentralDirectoryDataShouldParseWhen123456789ABCDEF0InLittleEndian() throws Exception {
+ testee.parseFromCentralDirectoryData(_123456789ABCDEF0_AS_LE_BYTE_ARRAY, 0, 8);
+ assertThat(testee.getSize())
+ .contains(0x123456789ABCDEF0L);
+ }
+
+ @Test
+ void parseFromCentralDirectoryDataShouldParseWhenFEDCBA9876543210InLittleEndian() throws Exception {
+ byte[] input = FEDCBA9876543210_AS_LE_BYTE_ARRAY;
+ testee.parseFromCentralDirectoryData(input, 0, 8);
+ assertThat(testee.getSize())
+ .contains(0xFEDCBA9876543210L);
+ }
+
+ @Test
+ void parseFromCentralDirectoryDataShouldHandleOffset() throws Exception {
+ byte[] input = Arrays.concatenate(UNUSED, _123456789ABCDEF0_AS_LE_BYTE_ARRAY);
+ testee.parseFromCentralDirectoryData(input, 2, 8);
+ assertThat(testee.getSize())
+ .contains(0x123456789ABCDEF0L);
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java
----------------------------------------------------------------------
diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java
index f3d1b89..4f268ec 100644
--- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java
+++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipArchiveEntryAssert.java
@@ -22,8 +22,10 @@ package org.apache.james.mailbox.backup;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipExtraField;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.apache.commons.io.IOUtils;
import org.assertj.core.api.AbstractAssert;
@@ -47,6 +49,15 @@ public class ZipArchiveEntryAssert extends AbstractAssert<ZipArchiveEntryAssert,
return new BasicErrorMessageFactory("%nExpecting %s to have content %s but was %s", zipArchiveEntry, expectedContent, actualContent);
}
+ private static BasicErrorMessageFactory shouldHaveExtraFields(ZipArchiveEntry zipArchiveEntry,
+ ZipExtraField[] expectedExtraFields,
+ ZipExtraField[] actualExtraFields) {
+ return new BasicErrorMessageFactory("%nExpecting %s to contain exactly being %s" +
+ " but was containing being %s", zipArchiveEntry,
+ Arrays.toString(expectedExtraFields),
+ Arrays.toString(actualExtraFields));
+ }
+
private final ZipFile zipFile;
private final ZipArchiveEntry actual;
@@ -76,4 +87,18 @@ public class ZipArchiveEntryAssert extends AbstractAssert<ZipArchiveEntryAssert,
}
return myself;
}
+
+ public ZipArchiveEntryAssert containsExactlyExtraFields(ZipExtraField... expectedExtraFields) {
+ isNotNull();
+ ZipExtraField[] actualExtraFields = actual.getExtraFields();
+ if (expectedExtraFields.length != actualExtraFields.length) {
+ throwAssertionError(shouldHaveExtraFields(actual, expectedExtraFields, actualExtraFields));
+ }
+ for (int i = 0; i < expectedExtraFields.length; i++) {
+ if (!expectedExtraFields[i].equals(actualExtraFields[i])) {
+ throwAssertionError(shouldHaveExtraFields(actual, expectedExtraFields, actualExtraFields));
+ }
+ }
+ return myself;
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/feb18013/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java
----------------------------------------------------------------------
diff --git a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java
index 888fed6..ed4e17a 100644
--- a/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java
+++ b/mailbox/backup/src/test/java/org/apache/james/mailbox/backup/ZipperTest.java
@@ -24,6 +24,7 @@ import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_CONT
import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_CONTENT_2;
import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_ID_1;
import static org.apache.james.mailbox.backup.MailboxMessageFixture.MESSAGE_ID_2;
+import static org.apache.james.mailbox.backup.MailboxMessageFixture.SIZE_1;
import static org.apache.james.mailbox.backup.ZipAssert.assertThatZip;
import java.io.File;
@@ -100,4 +101,16 @@ public class ZipperTest {
.hasStringContent(MESSAGE_CONTENT_2));
}
}
+
+ @Test
+ void archiveShouldWriteSizeMetadata() throws Exception {
+ testee.archive(ImmutableList.of(MESSAGE_1), new FileOutputStream(destination));
+
+ try (ZipFile zipFile = new ZipFile(destination)) {
+ assertThatZip(zipFile)
+ .containsExactlyEntriesMatching(
+ zipEntryAssert -> zipEntryAssert
+ .containsExactlyExtraFields(new SizeExtraField(SIZE_1)));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org