You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by rc...@apache.org on 2020/03/16 09:37:31 UTC
[james-project] 02/02: JAMES-3072: MailboxesBackup ExportService
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit e76b529a99d5c948807b401d02c51667f69273b5
Author: Tran Tien Duc <dt...@linagora.com>
AuthorDate: Wed Feb 19 11:51:31 2020 +0700
JAMES-3072: MailboxesBackup ExportService
---
pom.xml | 6 +
.../export/file/LocalFileBlobExportMechanism.java | 3 +-
server/protocols/webadmin/webadmin-mailbox/pom.xml | 34 +++
.../james/webadmin/service/ExportService.java | 136 ++++++++++
.../james/webadmin/service/ExportServiceTest.java | 273 +++++++++++++++++++++
5 files changed, 451 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index f83608b..869e985 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1125,6 +1125,12 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-file</artifactId>
+ <type>test-jar</type>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>blob-export-guice</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java b/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java
index c88d065..a2968ea 100644
--- a/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java
+++ b/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java
@@ -93,8 +93,9 @@ public class LocalFileBlobExportMechanism implements BlobExportMechanism {
private final DNSService dnsService;
private final Configuration configuration;
+ @VisibleForTesting
@Inject
- LocalFileBlobExportMechanism(MailetContext mailetContext, BlobStore blobStore, FileSystem fileSystem, DNSService dnsService, Configuration configuration) {
+ public LocalFileBlobExportMechanism(MailetContext mailetContext, BlobStore blobStore, FileSystem fileSystem, DNSService dnsService, Configuration configuration) {
this.mailetContext = mailetContext;
this.blobStore = blobStore;
this.fileSystem = fileSystem;
diff --git a/server/protocols/webadmin/webadmin-mailbox/pom.xml b/server/protocols/webadmin/webadmin-mailbox/pom.xml
index d189e00..cb3c161 100644
--- a/server/protocols/webadmin/webadmin-mailbox/pom.xml
+++ b/server/protocols/webadmin/webadmin-mailbox/pom.xml
@@ -101,6 +101,40 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>backup</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>backup</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>blob-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-file</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-file</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>blob-memory</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>james-json</artifactId>
<scope>test</scope>
<type>test-jar</type>
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java
new file mode 100644
index 0000000..e2f96ee
--- /dev/null
+++ b/server/protocols/webadmin/webadmin-mailbox/src/main/java/org/apache/james/webadmin/service/ExportService.java
@@ -0,0 +1,136 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.webadmin.service;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+
+import javax.inject.Inject;
+
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.export.api.BlobExportMechanism;
+import org.apache.james.blob.export.api.FileExtension;
+import org.apache.james.core.Username;
+import org.apache.james.mailbox.backup.MailboxBackup;
+import org.apache.james.task.Task;
+import org.apache.james.user.api.UsersRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.github.fge.lambdas.Throwing;
+
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+
+public class ExportService {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ExportService.class);
+ private static final String EXPLANATION = "The backup of your mailboxes has been exported to you";
+ private static final String FILE_PREFIX = "mailbox-backup-";
+
+ private final MailboxBackup mailboxBackup;
+ private final BlobStore blobStore;
+ private final BlobExportMechanism blobExport;
+ private final UsersRepository usersRepository;
+
+ @Inject
+ ExportService(MailboxBackup mailboxBackup, BlobStore blobStore, BlobExportMechanism blobExport, UsersRepository usersRepository) {
+ this.mailboxBackup = mailboxBackup;
+ this.blobStore = blobStore;
+ this.blobExport = blobExport;
+ this.usersRepository = usersRepository;
+ }
+
+ public Mono<Task.Result> export(Username username) {
+ return Mono.usingWhen(
+ Mono.fromCallable(() -> zipMailboxesContent(username)),
+ inputStream -> export(username, inputStream),
+ this::closeResource,
+ (inputStream, throwable) -> closeResource(inputStream),
+ this::closeResource
+ );
+ }
+
+ private InputStream zipMailboxesContent(Username username) throws IOException {
+ PipedOutputStream out = new PipedOutputStream();
+ PipedInputStream in = new PipedInputStream(out);
+
+ writeUserMailboxesContent(username, out)
+ .subscribeOn(Schedulers.elastic())
+ .subscribe();
+
+ return in;
+ }
+
+ private Mono<Task.Result> export(Username username, InputStream inputStream) {
+ return Mono.usingWhen(
+ blobStore.save(blobStore.getDefaultBucketName(), inputStream, BlobStore.StoragePolicy.LOW_COST),
+ blobId -> export(username, blobId),
+ this::deleteBlob)
+ .thenReturn(Task.Result.COMPLETED)
+ .onErrorResume(e -> {
+ LOGGER.error("Error exporting mailboxes of user: {}", username.asString(), e);
+ return Mono.just(Task.Result.PARTIAL);
+ });
+ }
+
+ private Mono<Void> export(Username username, BlobId blobId) {
+ return Mono.fromRunnable(Throwing.runnable(() ->
+ blobExport.blobId(blobId)
+ .with(usersRepository.getMailAddressFor(username))
+ .explanation(EXPLANATION)
+ .filePrefix(FILE_PREFIX + username.asString() + "-")
+ .fileExtension(FileExtension.ZIP)
+ .export())
+ .sneakyThrow());
+ }
+
+ private Mono<Void> deleteBlob(BlobId blobId) {
+ return Mono.from(blobStore.delete(blobStore.getDefaultBucketName(), blobId))
+ .onErrorResume(e -> {
+ LOGGER.error("Error deleting Blob with blobId: {}", blobId.asString(), e);
+ return Mono.empty();
+ });
+ }
+
+ private Mono<Void> writeUserMailboxesContent(Username username, PipedOutputStream out) {
+ return Mono.usingWhen(
+ Mono.fromCallable(() -> out),
+ outputStream -> Mono.fromRunnable(Throwing.runnable(() -> mailboxBackup.backupAccount(username, outputStream))),
+ this::closeResource,
+ (outputStream, throwable) -> closeResource(outputStream)
+ .doFinally(any -> LOGGER.error("Error while backing up mailboxes for user {}", username.asString(), throwable)),
+ this::closeResource);
+ }
+
+ private Mono<Void> closeResource(Closeable resource) {
+ return Mono.fromRunnable(() -> {
+ try {
+ resource.close();
+ } catch (IOException e) {
+ LOGGER.error("Error while closing resource", e);
+ }
+ });
+ }
+}
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java
new file mode 100644
index 0000000..ce15b01
--- /dev/null
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java
@@ -0,0 +1,273 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.webadmin.service;
+
+import static org.apache.james.mailbox.DefaultMailboxes.INBOX;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import javax.mail.MessagingException;
+
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.blob.api.ObjectNotFoundException;
+import org.apache.james.blob.export.api.BlobExportMechanism;
+import org.apache.james.blob.export.api.FileExtension;
+import org.apache.james.blob.export.file.FileSystemExtension;
+import org.apache.james.blob.export.file.LocalFileBlobExportMechanism;
+import org.apache.james.blob.memory.MemoryBlobStore;
+import org.apache.james.blob.memory.MemoryDumbBlobStore;
+import org.apache.james.core.Domain;
+import org.apache.james.core.Username;
+import org.apache.james.dnsservice.api.DNSService;
+import org.apache.james.domainlist.memory.MemoryDomainList;
+import org.apache.james.filesystem.api.FileSystem;
+import org.apache.james.mailbox.MailboxSession;
+import org.apache.james.mailbox.MessageManager;
+import org.apache.james.mailbox.backup.ArchiveService;
+import org.apache.james.mailbox.backup.DefaultMailboxBackup;
+import org.apache.james.mailbox.backup.MailArchivesLoader;
+import org.apache.james.mailbox.backup.MailboxBackup;
+import org.apache.james.mailbox.backup.ZipAssert;
+import org.apache.james.mailbox.backup.ZipMailArchiveRestorer;
+import org.apache.james.mailbox.backup.zip.ZipArchivesLoader;
+import org.apache.james.mailbox.backup.zip.Zipper;
+import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.inmemory.InMemoryMailboxManager;
+import org.apache.james.mailbox.inmemory.manager.InMemoryIntegrationResources;
+import org.apache.james.mailbox.model.ComposedMessageId;
+import org.apache.james.mailbox.model.MailboxPath;
+import org.apache.james.task.Task;
+import org.apache.james.user.memory.MemoryUsersRepository;
+import org.apache.mailet.base.MailAddressFixture;
+import org.apache.mailet.base.test.FakeMailContext;
+import org.assertj.core.api.SoftAssertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mockito;
+
+import com.google.common.base.Strings;
+import com.google.common.io.Files;
+
+import reactor.core.publisher.Mono;
+
+@ExtendWith(FileSystemExtension.class)
+class ExportServiceTest {
+
+ private static final String JAMES_HOST = "james-host";
+ private static final Domain DOMAIN = Domain.of("domain.tld");
+ private static final Username BOB = Username.fromLocalPartWithDomain("bob", DOMAIN);
+ private static final Username UNKNOWN_USER = Username.fromLocalPartWithDomain("unknown", DOMAIN);
+ private static final String PASSWORD = "password";
+ private static final String CORRESPONDING_FILE_HEADER = "corresponding-file";
+ private static final String MESSAGE_CONTENT = "MIME-Version: 1.0\r\n" +
+ "Subject: test\r\n" +
+ "Content-Type: text/plain; charset=UTF-8\r\n" +
+ "\r\n" +
+ "testmail";
+ private static final String TWELVE_MEGABYTES_STRING = Strings.repeat("0123456789\r\n", 1024 * 1024);
+ private static final String FILE_PREFIX = "mailbox-backup-";
+ private static final BlobId.Factory FACTORY = new HashBlobId.Factory();
+
+ private ExportService testee;
+ private InMemoryMailboxManager mailboxManager;
+ private MailboxSession bobSession;
+ private BlobStore blobStore;
+ private FakeMailContext mailetContext;
+
+ @BeforeEach
+ void setUp(FileSystem fileSystem) throws Exception {
+ mailboxManager = InMemoryIntegrationResources.defaultResources().getMailboxManager();
+ bobSession = mailboxManager.createSystemSession(BOB);
+
+ MailboxBackup backup = createMailboxBackup();
+ DNSService dnsService = createDnsService();
+ blobStore = Mockito.spy(new MemoryBlobStore(FACTORY, new MemoryDumbBlobStore()));
+ mailetContext = FakeMailContext.builder().postmaster(MailAddressFixture.POSTMASTER_AT_JAMES).build();
+ BlobExportMechanism blobExport = new LocalFileBlobExportMechanism(mailetContext, blobStore, fileSystem, dnsService,
+ LocalFileBlobExportMechanism.Configuration.DEFAULT_CONFIGURATION);
+ MemoryUsersRepository usersRepository = createUsersRepository(dnsService);
+
+ testee = new ExportService(backup, blobStore, blobExport, usersRepository);
+ }
+
+ private MemoryUsersRepository createUsersRepository(DNSService dnsService) throws Exception {
+ MemoryDomainList domainList = new MemoryDomainList(dnsService);
+ MemoryUsersRepository usersRepository = MemoryUsersRepository.withVirtualHosting(domainList);
+
+ domainList.addDomain(DOMAIN);
+ usersRepository.addUser(BOB, PASSWORD);
+ return usersRepository;
+ }
+
+ private DNSService createDnsService() throws UnknownHostException {
+ InetAddress localHost = mock(InetAddress.class);
+ when(localHost.getHostName()).thenReturn(JAMES_HOST);
+ DNSService dnsService = mock(DNSService.class);
+ when(dnsService.getLocalHost()).thenReturn(localHost);
+ return dnsService;
+ }
+
+ private DefaultMailboxBackup createMailboxBackup() {
+ ArchiveService archiveService = new Zipper();
+ MailArchivesLoader archiveLoader = new ZipArchivesLoader();
+ ZipMailArchiveRestorer archiveRestorer = new ZipMailArchiveRestorer(mailboxManager, archiveLoader);
+ return new DefaultMailboxBackup(mailboxManager, archiveService, archiveRestorer);
+ }
+
+ private String getFileUrl() throws MessagingException {
+ return mailetContext.getSentMails().get(0).getMsg().getHeader(CORRESPONDING_FILE_HEADER)[0];
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldReturnCompletedWhenUserDoesNotExist() {
+ assertThat(testee.export(UNKNOWN_USER).block())
+ .isEqualTo(Task.Result.COMPLETED);
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldReturnCompletedWhenExistingUserWithoutMailboxes() {
+ assertThat(testee.export(BOB).block())
+ .isEqualTo(Task.Result.COMPLETED);
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldReturnCompletedWhenExistingUser() throws Exception {
+ createAMailboxWithAMail(MESSAGE_CONTENT);
+
+ assertThat(testee.export(BOB).block())
+ .isEqualTo(Task.Result.COMPLETED);
+ }
+
+ private ComposedMessageId createAMailboxWithAMail(String message) throws MailboxException {
+ MailboxPath bobInboxPath = MailboxPath.inbox(BOB);
+ mailboxManager.createMailbox(bobInboxPath, bobSession);
+ return mailboxManager.getMailbox(bobInboxPath, bobSession)
+ .appendMessage(MessageManager.AppendCommand.builder()
+ .build(message),
+ bobSession);
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldProduceAnEmptyZipWhenUserDoesNotExist() throws Exception {
+ testee.export(UNKNOWN_USER).block();
+
+ ZipAssert.assertThatZip(new FileInputStream(getFileUrl()))
+ .hasNoEntry();
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldProduceAnEmptyZipWhenExistingUserWithoutAnyMailboxes() throws Exception {
+ testee.export(BOB).block();
+
+ ZipAssert.assertThatZip(new FileInputStream(getFileUrl()))
+ .hasNoEntry();
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldProduceAZipWithEntry() throws Exception {
+ ComposedMessageId id = createAMailboxWithAMail(MESSAGE_CONTENT);
+
+ testee.export(BOB).block();
+
+ ZipAssert.assertThatZip(new FileInputStream(getFileUrl()))
+ .containsOnlyEntriesMatching(
+ ZipAssert.EntryChecks.hasName(INBOX + "/").isDirectory(),
+ ZipAssert.EntryChecks.hasName(id.getMessageId().serialize()).hasStringContent(MESSAGE_CONTENT));
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldProduceAFileWithExpectedExtension() throws Exception {
+ createAMailboxWithAMail(MESSAGE_CONTENT);
+
+ testee.export(BOB).block();
+
+ assertThat(Files.getFileExtension(getFileUrl())).isEqualTo(FileExtension.ZIP.getExtension());
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldProduceAFileWithExpectedName() throws Exception {
+ createAMailboxWithAMail(MESSAGE_CONTENT);
+
+ testee.export(BOB).block();
+
+ File file = new File(getFileUrl());
+
+ assertThat(file.getName()).startsWith(FILE_PREFIX + BOB.asString());
+ }
+
+ @Test
+ void exportUserMailboxesWithSizableDataShouldProduceAFile() throws Exception {
+ ComposedMessageId id = createAMailboxWithAMail(TWELVE_MEGABYTES_STRING);
+
+ testee.export(BOB).block();
+
+ ZipAssert.assertThatZip(new FileInputStream(getFileUrl()))
+ .containsOnlyEntriesMatching(
+ ZipAssert.EntryChecks.hasName(INBOX + "/").isDirectory(),
+ ZipAssert.EntryChecks.hasName(id.getMessageId().serialize()).hasStringContent(TWELVE_MEGABYTES_STRING));
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldDeleteBlobAfterCompletion() throws Exception {
+ createAMailboxWithAMail(MESSAGE_CONTENT);
+
+ testee.export(BOB).block();
+
+ String fileName = Files.getNameWithoutExtension(getFileUrl());
+ String blobId = fileName.substring(fileName.lastIndexOf("-") + 1);
+
+ SoftAssertions.assertSoftly(softly -> {
+ assertThatThrownBy(() -> blobStore.read(blobStore.getDefaultBucketName(), FACTORY.from(blobId)))
+ .isInstanceOf(ObjectNotFoundException.class);
+ assertThatThrownBy(() -> blobStore.read(blobStore.getDefaultBucketName(), FACTORY.from(blobId)))
+ .hasMessage(String.format("blob '%s' not found in bucket '%s'", blobId, blobStore.getDefaultBucketName().asString()));
+ });
+ }
+
+ @Test
+ void exportUserMailboxesDataShouldReturnSuccessWhenBlobDeletingFails() throws Exception {
+ createAMailboxWithAMail(MESSAGE_CONTENT);
+
+ doReturn(Mono.error(new RuntimeException()))
+ .when(blobStore)
+ .delete(any(), any());
+
+ Task.Result result = testee.export(BOB).block();
+
+ String fileName = Files.getNameWithoutExtension(getFileUrl());
+ String blobId = fileName.substring(fileName.lastIndexOf("-") + 1);
+
+ blobStore.read(blobStore.getDefaultBucketName(), FACTORY.from(blobId));
+
+ assertThat(result).isEqualTo(Task.Result.COMPLETED);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org