You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2019/03/28 03:21:51 UTC
[james-project] 07/23: JAMES-2685 LocalFileBlobExportModule for
blob sharing
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit f52c3b63e6dd31b686445a3ac6644d5af23f3a95
Author: Tran Tien Duc <dt...@linagora.com>
AuthorDate: Thu Mar 21 11:39:11 2019 +0700
JAMES-2685 LocalFileBlobExportModule for blob sharing
to be used in integration tests
---
pom.xml | 10 ++++
.../export/file/LocalFileBlobExportMechanism.java | 7 +++
.../file/LocalFileBlobExportMechanismTest.java | 5 +-
.../pom.xml | 20 ++------
.../LocalFileBlobExportMechanismModule.java | 37 +++++++++++++++
server/container/guice/blob-memory-guice/pom.xml | 11 -----
server/container/guice/cassandra-guice/pom.xml | 4 ++
.../org/apache/james/CassandraJamesServerMain.java | 4 +-
server/container/guice/memory-guice/pom.xml | 4 ++
.../org/apache/james/MemoryJamesServerMain.java | 2 +
server/container/guice/pom.xml | 1 +
.../memory-jmap-integration-testing/pom.xml | 5 ++
.../vault/routes/DeletedMessagesVaultRoutes.java | 5 +-
.../james/webadmin/vault/routes/ExportService.java | 53 ++++++++++++----------
14 files changed, 109 insertions(+), 59 deletions(-)
diff --git a/pom.xml b/pom.xml
index f478bff..0a66da5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1103,6 +1103,16 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-guice</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-file</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>blob-memory</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java b/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java
index f731c9a..656dc45 100644
--- a/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java
+++ b/server/blob/blob-export-file/src/main/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanism.java
@@ -23,6 +23,8 @@ import java.io.File;
import java.io.IOException;
import java.net.UnknownHostException;
+import javax.inject.Inject;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.james.blob.api.BlobId;
@@ -43,6 +45,10 @@ public class LocalFileBlobExportMechanism implements BlobExportMechanism {
static final String CORRESPONDING_FILE_HEADER = "corresponding-file";
public static class Configuration {
+
+ private static final String DEFAULT_DIRECTORY_LOCATION = "file://var/blobExporting";
+ public static final Configuration DEFAULT_CONFIGURATION = new Configuration(DEFAULT_DIRECTORY_LOCATION);
+
private final String exportDirectory;
public Configuration(String exportDirectory) {
@@ -56,6 +62,7 @@ public class LocalFileBlobExportMechanism implements BlobExportMechanism {
private final DNSService dnsService;
private final Configuration configuration;
+ @Inject
LocalFileBlobExportMechanism(MailetContext mailetContext, BlobStore blobStore, FileSystem fileSystem, DNSService dnsService, Configuration configuration) {
this.mailetContext = mailetContext;
this.blobStore = blobStore;
diff --git a/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java b/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java
index 35983a0..83cd0e6 100644
--- a/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java
+++ b/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java
@@ -64,9 +64,8 @@ class LocalFileBlobExportMechanismTest {
DNSService dnsService = mock(DNSService.class);
when(dnsService.getLocalHost()).thenReturn(localHost);
- LocalFileBlobExportMechanism.Configuration blobExportConfiguration = new LocalFileBlobExportMechanism.Configuration("file://var/blobExporting");
-
- testee = new LocalFileBlobExportMechanism(mailetContext, blobStore, fileSystem, dnsService, blobExportConfiguration);
+ testee = new LocalFileBlobExportMechanism(mailetContext, blobStore, fileSystem, dnsService,
+ LocalFileBlobExportMechanism.Configuration.DEFAULT_CONFIGURATION);
}
@Test
diff --git a/server/container/guice/blob-memory-guice/pom.xml b/server/container/guice/blob-export-guice/pom.xml
similarity index 73%
copy from server/container/guice/blob-memory-guice/pom.xml
copy to server/container/guice/blob-export-guice/pom.xml
index fbe487e..29a358b 100644
--- a/server/container/guice/blob-memory-guice/pom.xml
+++ b/server/container/guice/blob-export-guice/pom.xml
@@ -28,35 +28,23 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>blob-memory-guice</artifactId>
+ <artifactId>blob-export-guice</artifactId>
<packaging>jar</packaging>
- <name>Apache James :: Server :: Blob Memory - guice injection</name>
- <description>Blob modules on memory storage</description>
+ <name>Apache James :: Server :: Blob Exporting Mechanisms - guice injection</name>
<dependencies>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>blob-api</artifactId>
+ <artifactId>blob-export-api</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>blob-memory</artifactId>
+ <artifactId>blob-export-file</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
<artifactId>james-server-guice-common</artifactId>
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <reuseForks>true</reuseForks>
- </configuration>
- </plugin>
- </plugins>
- </build>
</project>
\ No newline at end of file
diff --git a/server/container/guice/blob-export-guice/src/main/java/org/apache/james/modules/LocalFileBlobExportMechanismModule.java b/server/container/guice/blob-export-guice/src/main/java/org/apache/james/modules/LocalFileBlobExportMechanismModule.java
new file mode 100644
index 0000000..7d38201
--- /dev/null
+++ b/server/container/guice/blob-export-guice/src/main/java/org/apache/james/modules/LocalFileBlobExportMechanismModule.java
@@ -0,0 +1,37 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules;
+
+import org.apache.james.blob.export.api.BlobExportMechanism;
+import org.apache.james.blob.export.file.LocalFileBlobExportMechanism;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Scopes;
+
+public class LocalFileBlobExportMechanismModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(LocalFileBlobExportMechanism.Configuration.class).toInstance(LocalFileBlobExportMechanism.Configuration.DEFAULT_CONFIGURATION);
+
+ bind(LocalFileBlobExportMechanism.class).in(Scopes.SINGLETON);
+ bind(BlobExportMechanism.class).to(LocalFileBlobExportMechanism.class);
+ }
+}
diff --git a/server/container/guice/blob-memory-guice/pom.xml b/server/container/guice/blob-memory-guice/pom.xml
index fbe487e..e409945 100644
--- a/server/container/guice/blob-memory-guice/pom.xml
+++ b/server/container/guice/blob-memory-guice/pom.xml
@@ -48,15 +48,4 @@
<artifactId>james-server-guice-common</artifactId>
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <reuseForks>true</reuseForks>
- </configuration>
- </plugin>
- </plugins>
- </build>
</project>
\ No newline at end of file
diff --git a/server/container/guice/cassandra-guice/pom.xml b/server/container/guice/cassandra-guice/pom.xml
index 01127fa..206e9bd 100644
--- a/server/container/guice/cassandra-guice/pom.xml
+++ b/server/container/guice/cassandra-guice/pom.xml
@@ -112,6 +112,10 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>event-sourcing-event-store-cassandra</artifactId>
</dependency>
<dependency>
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
index a25e71d..f5f74d5 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
@@ -19,6 +19,7 @@
package org.apache.james;
+import org.apache.james.modules.LocalFileBlobExportMechanismModule;
import org.apache.james.modules.MailboxModule;
import org.apache.james.modules.activemq.ActiveMQQueueModule;
import org.apache.james.modules.data.CassandraDLPConfigurationStoreModule;
@@ -124,7 +125,8 @@ public class CassandraJamesServerMain {
CASSANDRA_SERVER_CORE_MODULE,
CASSANDRA_MAILBOX_MODULE,
PROTOCOLS,
- PLUGINS);
+ PLUGINS,
+ new LocalFileBlobExportMechanismModule());
public static void main(String[] args) throws Exception {
Configuration configuration = Configuration.builder()
diff --git a/server/container/guice/memory-guice/pom.xml b/server/container/guice/memory-guice/pom.xml
index 792c017..6e3ae2f 100644
--- a/server/container/guice/memory-guice/pom.xml
+++ b/server/container/guice/memory-guice/pom.xml
@@ -65,6 +65,10 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>blob-memory-guice</artifactId>
</dependency>
<dependency>
diff --git a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
index 6e0f235..c885f49 100644
--- a/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
+++ b/server/container/guice/memory-guice/src/main/java/org/apache/james/MemoryJamesServerMain.java
@@ -21,6 +21,7 @@ package org.apache.james;
import org.apache.commons.configuration.DefaultConfigurationBuilder;
import org.apache.james.modules.BlobMemoryModule;
+import org.apache.james.modules.LocalFileBlobExportMechanismModule;
import org.apache.james.modules.MailboxModule;
import org.apache.james.modules.data.MemoryDataJmapModule;
import org.apache.james.modules.data.MemoryDataModule;
@@ -82,6 +83,7 @@ public class MemoryJamesServerMain {
public static final Module IN_MEMORY_SERVER_MODULE = Modules.combine(
new BlobMemoryModule(),
new DeletedMessageVaultModule(),
+ new LocalFileBlobExportMechanismModule(),
new MailboxModule(),
new MemoryDataModule(),
new MemoryEventStoreModule(),
diff --git a/server/container/guice/pom.xml b/server/container/guice/pom.xml
index 5948b80..f0f0ef0 100644
--- a/server/container/guice/pom.xml
+++ b/server/container/guice/pom.xml
@@ -34,6 +34,7 @@
<modules>
<module>blob-api-guice</module>
+ <module>blob-export-guice</module>
<module>blob-memory-guice</module>
<module>blob-objectstorage-guice</module>
<module>cassandra-guice</module>
diff --git a/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml b/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml
index cf4901c..02531d3 100644
--- a/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml
+++ b/server/protocols/jmap-integration-testing/memory-jmap-integration-testing/pom.xml
@@ -52,6 +52,11 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-export-guice</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>james-server-dnsservice-test</artifactId>
</dependency>
<dependency>
diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java
index 940e3a8..7c1fa72 100644
--- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java
+++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutes.java
@@ -164,13 +164,12 @@ public class DeletedMessagesVaultRoutes implements Routes {
name = "exportTo",
paramType = "query",
example = "?exportTo=user@james.org",
- value = "Compulsory if action is export. Needs to be a valid mail address to represent for the destination " +
- "where deleted messages content is export to")
+ value = "Compulsory if action is export. Needs to be a valid mail address. The content of the vault matching the query will be sent to that address")
})
@ApiResponses(value = {
@ApiResponse(code = HttpStatus.CREATED_201, message = "Task is created", response = TaskIdDto.class),
@ApiResponse(code = HttpStatus.BAD_REQUEST_400, message = "Bad request - user param is invalid"),
- @ApiResponse(code = HttpStatus.NOT_FOUND_404, message = "Not found - requested user is not existed in the system"),
+ @ApiResponse(code = HttpStatus.NOT_FOUND_404, message = "Not found - requested user does not exist"),
@ApiResponse(code = HttpStatus.INTERNAL_SERVER_ERROR_500, message = "Internal server error - Something went bad on the server side.")
})
private TaskIdDto userActions(Request request, Response response) throws JsonExtractException {
diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java
index bab02b8..2e3c014 100644
--- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java
+++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java
@@ -19,11 +19,14 @@
package org.apache.james.webadmin.vault.routes;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
+import java.nio.file.Files;
import java.util.Collection;
+import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Stream;
@@ -38,8 +41,6 @@ import org.apache.james.vault.DeletedMessage;
import org.apache.james.vault.DeletedMessageVault;
import org.apache.james.vault.DeletedMessageZipper;
import org.apache.james.vault.search.Query;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.github.fge.lambdas.Throwing;
import com.github.fge.lambdas.functions.ThrowingFunction;
@@ -52,7 +53,17 @@ import reactor.core.scheduler.Schedulers;
class ExportService {
- private static final Logger LOGGER = LoggerFactory.getLogger(ExportService.class);
+ private class ZippedData {
+ private final long contentLength;
+ private final InputStream inputStream;
+
+ private ZippedData(long contentLength, InputStream content) {
+ this.contentLength = contentLength;
+ this.inputStream = content;
+ }
+ }
+
+ private static final String TEMPORARY_FILE_EXTENSION = ".temp";
private final BlobExportMechanism blobExport;
private final BlobStore blobStore;
@@ -75,8 +86,8 @@ class ExportService {
.doOnNext(any -> messageToExportCallback.run())
.collect(Guavate.toImmutableList())
.map(Collection::stream)
- .map(sneakyThrow(messages -> zipData(user, messages)))
- .flatMap(sneakyThrow(zippedStream -> blobStore.save(zippedStream, zippedStream.available())))
+ .flatMap(messages -> Mono.fromCallable(() -> zipData(user, messages)))
+ .flatMap(sneakyThrow(zippedData -> blobStore.save(zippedData.inputStream, zippedData.contentLength)))
.flatMap(blobId -> exportTo(user, exportToAddress, blobId))
.then();
}
@@ -86,27 +97,19 @@ class ExportService {
.publishOn(Schedulers.elastic());
}
- private PipedInputStream zipData(User user, Stream<DeletedMessage> messages) throws IOException {
- PipedOutputStream outputStream = new PipedOutputStream();
- PipedInputStream inputStream = new PipedInputStream();
- inputStream.connect(outputStream);
+ private ZippedData zipData(User user, Stream<DeletedMessage> messages) throws IOException {
+ File tempFile = temporaryFile();
+ FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
- asyncZipData(user, messages, outputStream).subscribe();
-
- return inputStream;
+ zipper.zip(message -> loadMessageContent(user, message), messages, fileOutputStream);
+ return new ZippedData(tempFile.length(), new FileInputStream(tempFile));
}
- private Mono<Void> asyncZipData(User user, Stream<DeletedMessage> messages, PipedOutputStream outputStream) {
- return Mono.fromRunnable(Throwing.runnable(() -> zipper.zip(message -> loadMessageContent(user, message), messages, outputStream)).sneakyThrow())
- .doOnSuccessOrError(Throwing.biConsumer((result, throwable) -> {
- if (throwable != null) {
- LOGGER.error("Error happens when zipping deleted messages", throwable);
- }
- outputStream.flush();
- outputStream.close();
- }))
- .subscribeOn(Schedulers.elastic())
- .then();
+ private File temporaryFile() throws IOException {
+ String tempFileName = UUID.randomUUID().toString();
+ File tempFile = Files.createTempFile(tempFileName, TEMPORARY_FILE_EXTENSION).toFile();
+ tempFile.deleteOnExit();
+ return tempFile;
}
private InputStream loadMessageContent(User user, DeletedMessage message) {
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org