You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/05/31 01:53:40 UTC
[01/14] james-project git commit: JAMES-2404 Use programmatically
registered extensions
Repository: james-project
Updated Branches:
refs/heads/master 006c81b4b -> 10dea04b2
JAMES-2404 Use programmatically registered extensions
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/f8faf50e
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/f8faf50e
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/f8faf50e
Branch: refs/heads/master
Commit: f8faf50e8cb76cdd1931ddcebb0f3e4865028eaf
Parents: d8ef8ae
Author: benwa <bt...@linagora.com>
Authored: Wed May 30 10:53:42 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:15 2018 +0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../apache/james/webadmin/WebAdminUtils.java | 8 +++
.../webadmin/routes/DomainQuotaRoutesTest.java | 21 ++++++-
.../ElasticSearchQuotaSearchExtension.java | 18 +++++-
.../webadmin/routes/GlobalQuotaRoutesTest.java | 17 +++++-
.../routes/ScanningQuotaSearchExtension.java | 30 +++++++++-
.../webadmin/routes/UserQuotaRoutesTest.java | 58 +++++++++++++++-----
.../routes/WebAdminQuotaSearchTestSystem.java | 38 ++-----------
8 files changed, 138 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6709cf9..3fd8a14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -617,7 +617,7 @@
<dnsjava.version>2.1.1</dnsjava.version>
<junit.version>4.11</junit.version>
- <junit.jupiter.version>5.0.2</junit.jupiter.version>
+ <junit.jupiter.version>5.2.0</junit.jupiter.version>
<junit.plateform.version>1.0.2</junit.plateform.version>
<junit.vintage.version>4.12.2</junit.vintage.version>
<jmock.version>2.6.0</jmock.version>
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-core/src/test/java/org/apache/james/webadmin/WebAdminUtils.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-core/src/test/java/org/apache/james/webadmin/WebAdminUtils.java b/server/protocols/webadmin/webadmin-core/src/test/java/org/apache/james/webadmin/WebAdminUtils.java
index f4f95ff..a0bf474 100644
--- a/server/protocols/webadmin/webadmin-core/src/test/java/org/apache/james/webadmin/WebAdminUtils.java
+++ b/server/protocols/webadmin/webadmin-core/src/test/java/org/apache/james/webadmin/WebAdminUtils.java
@@ -24,11 +24,13 @@ import static com.jayway.restassured.config.RestAssuredConfig.newConfig;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import org.apache.james.metrics.api.MetricFactory;
import org.apache.james.util.Port;
import org.apache.james.webadmin.authentication.NoAuthenticationFilter;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.jayway.restassured.builder.RequestSpecBuilder;
import com.jayway.restassured.http.ContentType;
@@ -36,6 +38,12 @@ import com.jayway.restassured.http.ContentType;
public class WebAdminUtils {
public static WebAdminServer createWebAdminServer(MetricFactory metricFactory, Routes... routes) throws IOException {
+ return createWebAdminServer(
+ metricFactory,
+ ImmutableList.copyOf(routes));
+ }
+
+ public static WebAdminServer createWebAdminServer(MetricFactory metricFactory, List<Routes> routes) throws IOException {
return new WebAdminServer(WebAdminConfiguration.TEST_CONFIGURATION,
ImmutableSet.copyOf(routes),
new NoAuthenticationFilter(),
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/DomainQuotaRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/DomainQuotaRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/DomainQuotaRoutesTest.java
index b27c0d6..2b68977 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/DomainQuotaRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/DomainQuotaRoutesTest.java
@@ -29,17 +29,34 @@ import org.apache.james.core.Domain;
import org.apache.james.mailbox.quota.MaxQuotaManager;
import org.apache.james.mailbox.quota.QuotaCount;
import org.apache.james.mailbox.quota.QuotaSize;
+import org.apache.james.quota.search.QuotaSearchTestSystem;
+import org.apache.james.webadmin.jackson.QuotaModule;
+import org.apache.james.webadmin.service.DomainQuotaService;
+import org.apache.james.webadmin.utils.JsonTransformer;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import com.google.common.collect.ImmutableSet;
import com.jayway.restassured.RestAssured;
import com.jayway.restassured.http.ContentType;
import com.jayway.restassured.path.json.JsonPath;
-@ExtendWith(ScanningQuotaSearchExtension.class)
class DomainQuotaRoutesTest {
+ @RegisterExtension
+ Extension scanningExtension = new ScanningQuotaSearchExtension(this::createDomainQuotaRoutes);
+
+ private DomainQuotaRoutes createDomainQuotaRoutes(QuotaSearchTestSystem testSystem) {
+ QuotaModule quotaModule = new QuotaModule();
+ return new DomainQuotaRoutes(
+ testSystem.getDomainList(),
+ new DomainQuotaService(testSystem.getMaxQuotaManager()),
+ testSystem.getUsersRepository(),
+ new JsonTransformer(quotaModule),
+ ImmutableSet.of(quotaModule));
+ }
private static final String QUOTA_DOMAINS = "/quota/domains";
private static final String PERDU_COM = "perdu.com";
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ElasticSearchQuotaSearchExtension.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ElasticSearchQuotaSearchExtension.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ElasticSearchQuotaSearchExtension.java
index d9a9e24..d5f5c3a 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ElasticSearchQuotaSearchExtension.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ElasticSearchQuotaSearchExtension.java
@@ -38,6 +38,9 @@ import org.apache.james.quota.search.elasticsearch.QuotaSearchIndexCreationUtil;
import org.apache.james.quota.search.elasticsearch.events.ElasticSearchQuotaMailboxListener;
import org.apache.james.quota.search.elasticsearch.json.QuotaRatioToElasticSearchJson;
import org.apache.james.user.memory.MemoryUsersRepository;
+import org.apache.james.webadmin.jackson.QuotaModule;
+import org.apache.james.webadmin.service.UserQuotaService;
+import org.apache.james.webadmin.utils.JsonTransformer;
import org.elasticsearch.client.Client;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
@@ -47,6 +50,9 @@ import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import org.junit.rules.TemporaryFolder;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
public class ElasticSearchQuotaSearchExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private WebAdminQuotaSearchTestSystem restQuotaSearchTestSystem;
@@ -91,7 +97,17 @@ public class ElasticSearchQuotaSearchExtension implements ParameterResolver, Bef
resources.getCurrentQuotaManager(),
() -> embeddedElasticSearch.awaitForElasticSearch());
- restQuotaSearchTestSystem = new WebAdminQuotaSearchTestSystem(quotaSearchTestSystem);
+ UserQuotaRoutes routes = new UserQuotaRoutes(
+ quotaSearchTestSystem.getUsersRepository(),
+ new UserQuotaService(
+ quotaSearchTestSystem.getMaxQuotaManager(),
+ quotaSearchTestSystem.getQuotaManager(),
+ quotaSearchTestSystem.getQuotaRootResolver(),
+ quotaSearchTestSystem.getQuotaSearcher()),
+ new JsonTransformer(new QuotaModule()),
+ ImmutableSet.of(new QuotaModule()));
+
+ restQuotaSearchTestSystem = new WebAdminQuotaSearchTestSystem(quotaSearchTestSystem, ImmutableList.of(routes));
} catch (Exception e) {
throw new ParameterResolutionException("Error while resolving parameter", e);
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/GlobalQuotaRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/GlobalQuotaRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/GlobalQuotaRoutesTest.java
index cb3380d..1b7e306 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/GlobalQuotaRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/GlobalQuotaRoutesTest.java
@@ -28,18 +28,31 @@ import java.util.Map;
import org.apache.james.mailbox.quota.MaxQuotaManager;
import org.apache.james.mailbox.quota.QuotaCount;
import org.apache.james.mailbox.quota.QuotaSize;
+import org.apache.james.quota.search.QuotaSearchTestSystem;
+import org.apache.james.webadmin.jackson.QuotaModule;
+import org.apache.james.webadmin.service.GlobalQuotaService;
+import org.apache.james.webadmin.utils.JsonTransformer;
import org.assertj.core.api.SoftAssertions;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.RegisterExtension;
import com.jayway.restassured.RestAssured;
import com.jayway.restassured.http.ContentType;
import com.jayway.restassured.path.json.JsonPath;
-@ExtendWith(ScanningQuotaSearchExtension.class)
class GlobalQuotaRoutesTest {
+ @RegisterExtension
+ Extension scanningExtension = new ScanningQuotaSearchExtension(this::createGlobalQuotaRoutes);
+
+ private GlobalQuotaRoutes createGlobalQuotaRoutes(QuotaSearchTestSystem testSystem) {
+ return new GlobalQuotaRoutes(
+ new GlobalQuotaService(testSystem.getMaxQuotaManager()),
+ new JsonTransformer(new QuotaModule()));
+ }
+
private MaxQuotaManager maxQuotaManager;
@BeforeEach
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ScanningQuotaSearchExtension.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ScanningQuotaSearchExtension.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ScanningQuotaSearchExtension.java
index b595181..9a2a80a 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ScanningQuotaSearchExtension.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/ScanningQuotaSearchExtension.java
@@ -21,6 +21,9 @@ package org.apache.james.webadmin.routes;
import static org.mockito.Mockito.mock;
+import java.util.List;
+import java.util.function.Function;
+
import org.apache.commons.configuration.DefaultConfigurationBuilder;
import org.apache.james.dnsservice.api.DNSService;
import org.apache.james.domainlist.memory.MemoryDomainList;
@@ -30,6 +33,7 @@ import org.apache.james.quota.search.QuotaSearchTestSystem;
import org.apache.james.quota.search.scanning.ClauseConverter;
import org.apache.james.quota.search.scanning.ScanningQuotaSearcher;
import org.apache.james.user.memory.MemoryUsersRepository;
+import org.apache.james.webadmin.Routes;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
@@ -37,10 +41,20 @@ import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
+import com.github.steveash.guavate.Guavate;
+import com.google.common.collect.ImmutableList;
+import com.jayway.restassured.specification.RequestSpecification;
+
public class ScanningQuotaSearchExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Runnable NO_AWAIT = () -> {};
+ private final List<Function<QuotaSearchTestSystem, Routes>> routesGenerators;
private WebAdminQuotaSearchTestSystem restQuotaSearchTestSystem;
+ private QuotaSearchTestSystem quotaSearchTestSystem;
+
+ public ScanningQuotaSearchExtension(Function<QuotaSearchTestSystem, Routes>... routesGenerators) {
+ this.routesGenerators = ImmutableList.copyOf(routesGenerators);
+ }
@Override
public void beforeEach(ExtensionContext context) {
@@ -54,7 +68,7 @@ public class ScanningQuotaSearchExtension implements ParameterResolver, BeforeEa
domainList.configure(new DefaultConfigurationBuilder());
usersRepository.setDomainList(domainList);
- QuotaSearchTestSystem quotaSearchTestSystem = new QuotaSearchTestSystem(
+ quotaSearchTestSystem = new QuotaSearchTestSystem(
resources.getMaxQuotaManager(),
resources.getMailboxManager(),
resources.getQuotaManager(),
@@ -66,7 +80,11 @@ public class ScanningQuotaSearchExtension implements ParameterResolver, BeforeEa
resources.getCurrentQuotaManager(),
NO_AWAIT);
- restQuotaSearchTestSystem = new WebAdminQuotaSearchTestSystem(quotaSearchTestSystem);
+ List<Routes> routes = routesGenerators.stream()
+ .map(generator -> generator.apply(quotaSearchTestSystem))
+ .collect(Guavate.toImmutableList());
+
+ restQuotaSearchTestSystem = new WebAdminQuotaSearchTestSystem(quotaSearchTestSystem, routes);
} catch (Exception e) {
throw new ParameterResolutionException("Error while resolving parameter", e);
}
@@ -86,4 +104,12 @@ public class ScanningQuotaSearchExtension implements ParameterResolver, BeforeEa
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
return restQuotaSearchTestSystem;
}
+
+ public QuotaSearchTestSystem getQuotaSearchTestSystem() {
+ return quotaSearchTestSystem;
+ }
+
+ public RequestSpecification getRequestSpecification() {
+ return restQuotaSearchTestSystem.getRequestSpecification();
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/UserQuotaRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/UserQuotaRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/UserQuotaRoutesTest.java
index 694cea4..d521b5a 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/UserQuotaRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/UserQuotaRoutesTest.java
@@ -48,15 +48,20 @@ import org.apache.james.mailbox.quota.QuotaSize;
import org.apache.james.mailbox.quota.UserQuotaRootResolver;
import org.apache.james.quota.search.QuotaSearchTestSystem;
import org.apache.james.user.api.UsersRepository;
+import org.apache.james.webadmin.jackson.QuotaModule;
+import org.apache.james.webadmin.service.UserQuotaService;
+import org.apache.james.webadmin.utils.JsonTransformer;
import org.assertj.core.api.SoftAssertions;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.Extension;
+import org.junit.jupiter.api.extension.RegisterExtension;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
import com.jayway.restassured.RestAssured;
import com.jayway.restassured.http.ContentType;
import com.jayway.restassured.path.json.JsonPath;
@@ -75,6 +80,20 @@ class UserQuotaRoutesTest {
private static final String COUNT = "count";
private static final String SIZE = "size";
+ private final ScanningQuotaSearchExtension scanningExtension = new ScanningQuotaSearchExtension(this::createUserQuotaRoutes);
+
+ private UserQuotaRoutes createUserQuotaRoutes(QuotaSearchTestSystem testSystem) {
+ QuotaModule quotaModule = new QuotaModule();
+ return new UserQuotaRoutes(testSystem.getUsersRepository(),
+ new UserQuotaService(
+ testSystem.getMaxQuotaManager(),
+ testSystem.getQuotaManager(),
+ testSystem.getQuotaRootResolver(),
+ testSystem.getQuotaSearcher()),
+ new JsonTransformer(quotaModule),
+ ImmutableSet.of(quotaModule));
+ }
+
@BeforeEach
public void setUp(WebAdminQuotaSearchTestSystem testSystem) throws Exception {
DomainList domainList = testSystem.getQuotaSearchTestSystem().getDomainList();
@@ -450,20 +469,21 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class ScanningGetUsersQuotaRouteTest implements GetUsersQuotaRouteContract {
-
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
}
@Nested
- @ExtendWith(ElasticSearchQuotaSearchExtension.class)
class ElasticSearchGetUsersQuotaRouteTest implements GetUsersQuotaRouteContract {
-
+ @RegisterExtension
+ Extension registeredExtension = new ElasticSearchQuotaSearchExtension();
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class GetCount {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
@Test
void getCountShouldReturnNotFoundWhenUserDoesntExist() {
@@ -503,8 +523,10 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class GetSize {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
+
@Test
void getSizeShouldReturnNotFoundWhenUserDoesntExist() {
when()
@@ -544,8 +566,10 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class PutCount {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
+
@Test
void putCountShouldReturnNotFoundWhenUserDoesntExist() {
given()
@@ -650,8 +674,10 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class PutSize {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
+
@Test
void putSizeAcceptEscapedUsers() {
given()
@@ -742,8 +768,9 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class DeleteCount {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
@Test
void deleteCountShouldReturnNotFoundWhenUserDoesntExist() {
@@ -769,8 +796,10 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class DeleteSize {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
+
@Test
void deleteSizeShouldReturnNotFoundWhenUserDoesntExist() {
when()
@@ -796,8 +825,10 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class GetQuota {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
+
@Test
void getQuotaShouldReturnNotFoundWhenUserDoesntExist() {
when()
@@ -1042,8 +1073,9 @@ class UserQuotaRoutesTest {
}
@Nested
- @ExtendWith(ScanningQuotaSearchExtension.class)
class PutQuota {
+ @RegisterExtension
+ Extension registeredExtension = scanningExtension;
@Test
void putQuotaShouldReturnNotFoundWhenUserDoesntExist() {
http://git-wip-us.apache.org/repos/asf/james-project/blob/f8faf50e/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/WebAdminQuotaSearchTestSystem.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/WebAdminQuotaSearchTestSystem.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/WebAdminQuotaSearchTestSystem.java
index 75f8d26..d9a1893 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/WebAdminQuotaSearchTestSystem.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/routes/WebAdminQuotaSearchTestSystem.java
@@ -21,17 +21,14 @@ package org.apache.james.webadmin.routes;
import static org.apache.james.webadmin.WebAdminServer.NO_CONFIGURATION;
+import java.util.List;
+
import org.apache.james.metrics.api.NoopMetricFactory;
import org.apache.james.quota.search.QuotaSearchTestSystem;
+import org.apache.james.webadmin.Routes;
import org.apache.james.webadmin.WebAdminServer;
import org.apache.james.webadmin.WebAdminUtils;
-import org.apache.james.webadmin.jackson.QuotaModule;
-import org.apache.james.webadmin.service.DomainQuotaService;
-import org.apache.james.webadmin.service.GlobalQuotaService;
-import org.apache.james.webadmin.service.UserQuotaService;
-import org.apache.james.webadmin.utils.JsonTransformer;
-import com.google.common.collect.ImmutableSet;
import com.jayway.restassured.specification.RequestSpecification;
public class WebAdminQuotaSearchTestSystem {
@@ -39,34 +36,9 @@ public class WebAdminQuotaSearchTestSystem {
private final WebAdminServer webAdminServer;
private final RequestSpecification requestSpecBuilder;
- public WebAdminQuotaSearchTestSystem(QuotaSearchTestSystem quotaSearchTestSystem) throws Exception {
+ public WebAdminQuotaSearchTestSystem(QuotaSearchTestSystem quotaSearchTestSystem, List<Routes> routes) throws Exception {
this.quotaSearchTestSystem = quotaSearchTestSystem;
-
- UserQuotaService userQuotaService = new UserQuotaService(quotaSearchTestSystem.getMaxQuotaManager(),
- quotaSearchTestSystem.getQuotaManager(),
- quotaSearchTestSystem.getQuotaRootResolver(),
- quotaSearchTestSystem.getQuotaSearcher());
-
- QuotaModule quotaModule = new QuotaModule();
- JsonTransformer jsonTransformer = new JsonTransformer(quotaModule);
- UserQuotaRoutes userQuotaRoutes = new UserQuotaRoutes(quotaSearchTestSystem.getUsersRepository(),
- userQuotaService, jsonTransformer,
- ImmutableSet.of(quotaModule));
- DomainQuotaRoutes domainQuotaRoutes = new DomainQuotaRoutes(
- quotaSearchTestSystem.getDomainList(),
- new DomainQuotaService(quotaSearchTestSystem.getMaxQuotaManager()),
- quotaSearchTestSystem.getUsersRepository(),
- jsonTransformer,
- ImmutableSet.of(quotaModule));
- GlobalQuotaRoutes globalQuotaRoutes = new GlobalQuotaRoutes(
- new GlobalQuotaService(quotaSearchTestSystem.getMaxQuotaManager()),
- jsonTransformer);
-
- this.webAdminServer = WebAdminUtils.createWebAdminServer(
- new NoopMetricFactory(),
- userQuotaRoutes,
- domainQuotaRoutes,
- globalQuotaRoutes);
+ this.webAdminServer = WebAdminUtils.createWebAdminServer(new NoopMetricFactory(), routes);
this.webAdminServer.configure(NO_CONFIGURATION);
this.webAdminServer.await();
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[09/14] james-project git commit: MAILBOX-338 Allow metric to specify
increment/decrement count
Posted by bt...@apache.org.
MAILBOX-338 Allow metric to specify increment/decrement count
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/918c86ff
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/918c86ff
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/918c86ff
Branch: refs/heads/master
Commit: 918c86ff8bb9aa88fc9aef5b0c4208b62922b38f
Parents: d0fd664
Author: benwa <bt...@linagora.com>
Authored: Sun May 27 11:31:13 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../main/java/org/apache/james/metrics/api/Metric.java | 4 ++++
.../org/apache/james/metrics/api/NoopMetricFactory.java | 9 ++++++++-
.../apache/james/metrics/dropwizard/DropWizardMetric.java | 10 ++++++++++
.../org/apache/james/metrics/logger/DefaultMetric.java | 9 +++++++++
4 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/918c86ff/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Metric.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Metric.java b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Metric.java
index b8b552f..1e0aa1a 100644
--- a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Metric.java
+++ b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Metric.java
@@ -25,4 +25,8 @@ public interface Metric {
void decrement();
+ void add(int value);
+
+ void remove(int value);
+
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/918c86ff/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
index f421075..f5d82ca 100644
--- a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
+++ b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
@@ -36,7 +36,14 @@ public class NoopMetricFactory implements MetricFactory {
@Override
public void decrement() {
}
-
+
+ @Override
+ public void add(int value) {
+ }
+
+ @Override
+ public void remove(int value) {
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/james-project/blob/918c86ff/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetric.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetric.java b/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetric.java
index 1e55395..3aef262 100644
--- a/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetric.java
+++ b/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardMetric.java
@@ -40,4 +40,14 @@ public class DropWizardMetric implements Metric {
public void decrement() {
counter.dec();
}
+
+ @Override
+ public void add(int value) {
+ counter.inc(value);
+ }
+
+ @Override
+ public void remove(int value) {
+ counter.dec(value);
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/918c86ff/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetric.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetric.java b/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetric.java
index 3c80e4d..90697fa 100644
--- a/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetric.java
+++ b/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetric.java
@@ -40,4 +40,13 @@ public class DefaultMetric implements Metric {
value.decrementAndGet();
}
+ @Override
+ public void add(int i) {
+ value.addAndGet(i);
+ }
+
+ @Override
+ public void remove(int i) {
+ value.addAndGet(-1 * i);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[10/14] james-project git commit: MAILBOX-338 Test
CachingTextExtractor concurrency
Posted by bt...@apache.org.
MAILBOX-338 Test CachingTextExtractor concurrency
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/f63e5d87
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/f63e5d87
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/f63e5d87
Branch: refs/heads/master
Commit: f63e5d87cc3928ed1d6d95807c4b3e9044ec1aeb
Parents: 6b00971
Author: benwa <bt...@linagora.com>
Authored: Tue May 29 17:36:29 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../james/mailbox/tika/CachingTextExtractor.java | 18 ++++++++++++------
.../mailbox/tika/CachingTextExtractorTest.java | 15 +++++++++++++++
2 files changed, 27 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/f63e5d87/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
index e2e122b..b0bae1f 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -23,6 +23,7 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.time.Duration;
import java.util.Optional;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.commons.codec.digest.DigestUtils;
@@ -114,14 +115,19 @@ public class CachingTextExtractor implements TextExtractor {
byte[] bytes = IOUtils.toByteArray(inputStream);
String key = DigestUtils.sha256Hex(bytes);
- ParsedContent cachedValue = cache.getIfPresent(key);
- if (cachedValue != null) {
- return cachedValue;
+ try {
+ return cache.get(key,
+ () -> underlying.extractContent(new ByteArrayInputStream(bytes), contentType));
+ } catch (ExecutionException e) {
+ throw unwrap(e);
}
+ }
- ParsedContent realValue = underlying.extractContent(new ByteArrayInputStream(bytes), contentType);
- cache.put(key, realValue);
- return realValue;
+ private Exception unwrap(Exception e) {
+ return Optional.ofNullable(e.getCause())
+ .filter(throwable -> throwable instanceof Exception)
+ .map(throwable -> (Exception) throwable)
+ .orElse(e);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/james-project/blob/f63e5d87/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
index 5c0ebad..637bedb 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -34,6 +34,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.IntStream;
@@ -42,7 +43,9 @@ import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.metrics.api.NoopGaugeRegistry;
import org.apache.james.metrics.api.NoopMetricFactory;
+import org.apache.james.util.concurrency.ConcurrentTestRunner;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
import com.github.fge.lambdas.Throwing;
@@ -179,4 +182,16 @@ public class CachingTextExtractorTest {
verifyZeroInteractions(wrappedTextExtractor);
}
+ @RepeatedTest(10)
+ void concurrentValueComputationShouldNotLeadToDuplicatedBackendAccess() throws Exception {
+ int concurrentThreadCount = 10;
+ int operationCount = 1;
+ new ConcurrentTestRunner(concurrentThreadCount, operationCount,
+ (a, b) -> textExtractor.extractContent(INPUT_STREAM.get(), CONTENT_TYPE))
+ .run()
+ .awaitTermination(1, TimeUnit.MINUTES);
+
+ verify(wrappedTextExtractor, times(1)).extractContent(any(), any());
+ }
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[07/14] james-project git commit: MAILBOX-338 ValueWithUnit is only
used for size
Posted by bt...@apache.org.
MAILBOX-338 ValueWithUnit is only used for size
We should:
- Move it to a more generic package
- Rename the class and method to make it size specific
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/e3751bea
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/e3751bea
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/e3751bea
Branch: refs/heads/master
Commit: e3751bea31a93470e7bff47f21b162bbef8d9569
Parents: 905e942
Author: benwa <bt...@linagora.com>
Authored: Fri May 25 08:57:54 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
server/container/cli/pom.xml | 4 +
.../java/org/apache/james/cli/ServerCmd.java | 24 ++--
.../apache/james/cli/utils/ValueWithUnit.java | 128 -------------------
.../james/cli/utils/ValueWithUnitTest.java | 73 -----------
.../main/java/org/apache/james/util/Size.java | 128 +++++++++++++++++++
.../java/org/apache/james/util/SizeTest.java | 73 +++++++++++
6 files changed, 217 insertions(+), 213 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/e3751bea/server/container/cli/pom.xml
----------------------------------------------------------------------
diff --git a/server/container/cli/pom.xml b/server/container/cli/pom.xml
index fe215fc..a6a0845 100644
--- a/server/container/cli/pom.xml
+++ b/server/container/cli/pom.xml
@@ -46,6 +46,10 @@
<artifactId>james-server-mailbox-adapter</artifactId>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>james-server-util</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/james-project/blob/e3751bea/server/container/cli/src/main/java/org/apache/james/cli/ServerCmd.java
----------------------------------------------------------------------
diff --git a/server/container/cli/src/main/java/org/apache/james/cli/ServerCmd.java b/server/container/cli/src/main/java/org/apache/james/cli/ServerCmd.java
index 19f0981..c6ac5e4 100644
--- a/server/container/cli/src/main/java/org/apache/james/cli/ServerCmd.java
+++ b/server/container/cli/src/main/java/org/apache/james/cli/ServerCmd.java
@@ -44,7 +44,6 @@ import org.apache.james.cli.probe.impl.JmxMailboxProbe;
import org.apache.james.cli.probe.impl.JmxQuotaProbe;
import org.apache.james.cli.probe.impl.JmxSieveProbe;
import org.apache.james.cli.type.CmdType;
-import org.apache.james.cli.utils.ValueWithUnit;
import org.apache.james.mailbox.quota.QuotaCount;
import org.apache.james.mailbox.quota.QuotaSize;
import org.apache.james.mailbox.quota.QuotaValue;
@@ -56,6 +55,7 @@ import org.apache.james.mailbox.store.probe.SieveProbe;
import org.apache.james.probe.DataProbe;
import org.apache.james.rrt.lib.Mappings;
import org.apache.james.util.Port;
+import org.apache.james.util.Size;
import org.apache.james.util.SizeFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -312,10 +312,10 @@ public class ServerCmd {
mailboxProbe.reIndexAll();
break;
case SETSIEVEQUOTA:
- sieveProbe.setSieveQuota(ValueWithUnit.parse(arguments[1]).getConvertedValue());
+ sieveProbe.setSieveQuota(Size.parse(arguments[1]).asBytes());
break;
case SETSIEVEUSERQUOTA:
- sieveProbe.setSieveQuota(arguments[1], ValueWithUnit.parse(arguments[2]).getConvertedValue());
+ sieveProbe.setSieveQuota(arguments[1], Size.parse(arguments[2]).asBytes());
break;
case GETSIEVEQUOTA:
printStream.println("Storage space allowed for Sieve scripts by default: "
@@ -339,7 +339,7 @@ public class ServerCmd {
}
private SerializableQuotaValue<QuotaSize> parseQuotaSize(String argument) throws Exception {
- long convertedValue = ValueWithUnit.parse(argument).getConvertedValue();
+ long convertedValue = Size.parse(argument).asBytes();
return longToSerializableQuotaValue(convertedValue, QuotaSize.unlimited(), QuotaSize::size);
}
@@ -388,10 +388,10 @@ public class ServerCmd {
private String formatStorageValue(Long value) {
if (value == null) {
- return ValueWithUnit.UNKNOWN;
+ return Size.UNKNOWN;
}
if (value == SerializableQuota.UNLIMITED) {
- return ValueWithUnit.UNLIMITED;
+ return Size.UNLIMITED;
}
return SizeFormat.format(value);
}
@@ -401,18 +401,18 @@ public class ServerCmd {
.toValue(QuotaSize::size, QuotaSize.unlimited())
.map(size -> {
if (size.isUnlimited()) {
- return ValueWithUnit.UNLIMITED;
+ return Size.UNLIMITED;
}
return SizeFormat.format(size.asLong());
- }).orElse(ValueWithUnit.UNKNOWN);
+ }).orElse(Size.UNKNOWN);
}
private String formatMessageValue(Long value) {
if (value == null) {
- return ValueWithUnit.UNKNOWN;
+ return Size.UNKNOWN;
}
if (value == SerializableQuota.UNLIMITED) {
- return ValueWithUnit.UNLIMITED;
+ return Size.UNLIMITED;
}
return String.valueOf(value);
}
@@ -422,10 +422,10 @@ public class ServerCmd {
.toValue(QuotaCount::count, QuotaCount.unlimited())
.map(count -> {
if (count.isUnlimited()) {
- return ValueWithUnit.UNLIMITED;
+ return Size.UNLIMITED;
}
return String.valueOf(count.asLong());
- }).orElse(ValueWithUnit.UNKNOWN);
+ }).orElse(Size.UNKNOWN);
}
private void print(Map<String, Mappings> map, PrintStream out) {
http://git-wip-us.apache.org/repos/asf/james-project/blob/e3751bea/server/container/cli/src/main/java/org/apache/james/cli/utils/ValueWithUnit.java
----------------------------------------------------------------------
diff --git a/server/container/cli/src/main/java/org/apache/james/cli/utils/ValueWithUnit.java b/server/container/cli/src/main/java/org/apache/james/cli/utils/ValueWithUnit.java
deleted file mode 100644
index b61d653..0000000
--- a/server/container/cli/src/main/java/org/apache/james/cli/utils/ValueWithUnit.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.cli.utils;
-
-import com.google.common.math.LongMath;
-
-/**
- * This class is an helper for parsing integer input that may contain units.
- */
-public class ValueWithUnit {
-
- public static final String UNKNOWN = "UNKNOWN";
- public static final String UNLIMITED = "UNLIMITED";
- public static final long UNKNOWN_VALUE = Long.MIN_VALUE;
- public static final long UNLIMITED_VALUE = -1;
-
- /**
- * supported units : B ( 2^0 ), K ( 2^10 ), M ( 2^20 ), G ( 2^30 )
- * See RFC822.SIZE
- */
- private enum Unit {
- NoUnit,
- B,
- K,
- M,
- G
- }
-
- private static final long base = 1024;
-
- Unit unit;
- Long value;
-
- private ValueWithUnit(Unit unit, Long value) {
- this.unit = unit;
- this.value = value;
- }
-
- public static ValueWithUnit parse(String providedLongWithUnitString) throws Exception {
- if (providedLongWithUnitString.equalsIgnoreCase(UNKNOWN)) {
- return new ValueWithUnit(Unit.NoUnit, UNKNOWN_VALUE);
- }
- if (providedLongWithUnitString.equalsIgnoreCase(UNLIMITED)) {
- return new ValueWithUnit(Unit.NoUnit, UNLIMITED_VALUE);
- }
- char lastChar = providedLongWithUnitString.charAt(providedLongWithUnitString.length() - 1);
- Unit unit = getUnit(lastChar);
- String argWithoutUnit = removeLastCharIfNeeded(providedLongWithUnitString, unit);
- return new ValueWithUnit(unit, Long.parseLong(argWithoutUnit));
- }
-
- public Unit getUnit() {
- return unit;
- }
-
- public Long getValue() {
- return value;
- }
-
- public long getConvertedValue() {
- switch (unit) {
- case G:
- return value * LongMath.pow(base, 3);
- case M:
- return value * LongMath.pow(base, 2);
- case K:
- return value * LongMath.pow(base, 1);
- default:
- return value;
- }
- }
-
- private static String removeLastCharIfNeeded(String providedLongWithUnitString, Unit unit) {
- if (unit != Unit.NoUnit) {
- return providedLongWithUnitString.substring(0, providedLongWithUnitString.length() - 1);
- } else {
- return providedLongWithUnitString;
- }
- }
-
- private static Unit getUnit(char lastChar) throws Exception {
- switch (lastChar) {
- case 'K' :
- case 'k' :
- return Unit.K;
- case 'M' :
- case 'm' :
- return Unit.M;
- case 'G' :
- case 'g' :
- return Unit.G;
- case 'b' :
- case 'B' :
- return Unit.B;
- case '1' :
- case '2' :
- case '3' :
- case '4' :
- case '5' :
- case '6' :
- case '7' :
- case '8' :
- case '9' :
- case '0' :
- return Unit.NoUnit;
- default:
- throw new Exception("No unit corresponding to char : " + lastChar);
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/e3751bea/server/container/cli/src/test/java/org/apache/james/cli/utils/ValueWithUnitTest.java
----------------------------------------------------------------------
diff --git a/server/container/cli/src/test/java/org/apache/james/cli/utils/ValueWithUnitTest.java b/server/container/cli/src/test/java/org/apache/james/cli/utils/ValueWithUnitTest.java
deleted file mode 100644
index fe897dc..0000000
--- a/server/container/cli/src/test/java/org/apache/james/cli/utils/ValueWithUnitTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.cli.utils;
-
-import static org.assertj.core.api.Assertions.assertThat;
-
-import org.junit.Test;
-
-public class ValueWithUnitTest {
-
- @Test
- public void testNoUnit() throws Exception {
- assertThat(ValueWithUnit.parse("1024").getConvertedValue()).isEqualTo(1024);
- }
-
- @Test
- public void testUnitB() throws Exception {
- assertThat(ValueWithUnit.parse("1024B").getConvertedValue()).isEqualTo(1024);
- }
-
- @Test
- public void testUnitK() throws Exception {
- assertThat(ValueWithUnit.parse("5K").getConvertedValue()).isEqualTo(5 * 1024);
- }
-
- @Test
- public void testUnitM() throws Exception {
- assertThat(ValueWithUnit.parse("5M").getConvertedValue()).isEqualTo(5 * 1024 * 1024);
- }
-
- @Test
- public void testUnitG() throws Exception {
- assertThat(ValueWithUnit.parse("1G").getConvertedValue()).isEqualTo(1024 * 1024 * 1024);
- }
-
- @Test
- public void testUnknown() throws Exception {
- assertThat(ValueWithUnit.parse("unknown").getConvertedValue()).isEqualTo(ValueWithUnit.UNKNOWN_VALUE);
- }
-
- @Test
- public void testUnlimited() throws Exception {
- assertThat(ValueWithUnit.parse("unlimited").getConvertedValue()).isEqualTo(ValueWithUnit.UNLIMITED_VALUE);
- }
-
- @Test(expected = Exception.class)
- public void testBadUnit() throws Exception {
- ValueWithUnit.parse("42T");
- }
-
- @Test(expected = NumberFormatException.class)
- public void testWrongNumber() throws Exception {
- ValueWithUnit.parse("42RG");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/james-project/blob/e3751bea/server/container/util/src/main/java/org/apache/james/util/Size.java
----------------------------------------------------------------------
diff --git a/server/container/util/src/main/java/org/apache/james/util/Size.java b/server/container/util/src/main/java/org/apache/james/util/Size.java
new file mode 100644
index 0000000..341a0e7
--- /dev/null
+++ b/server/container/util/src/main/java/org/apache/james/util/Size.java
@@ -0,0 +1,128 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.util;
+
+import com.google.common.math.LongMath;
+
+/**
+ * This class is an helper for parsing integer input that may contain units.
+ */
+public class Size {
+
+ public static final String UNKNOWN = "UNKNOWN";
+ public static final String UNLIMITED = "UNLIMITED";
+ public static final long UNKNOWN_VALUE = Long.MIN_VALUE;
+ public static final long UNLIMITED_VALUE = -1;
+
+ /**
+ * supported units : B ( 2^0 ), K ( 2^10 ), M ( 2^20 ), G ( 2^30 )
+ * See RFC822.SIZE
+ */
+ private enum Unit {
+ NoUnit,
+ B,
+ K,
+ M,
+ G
+ }
+
+ private static final long base = 1024;
+
+ Unit unit;
+ Long value;
+
+ private Size(Unit unit, Long value) {
+ this.unit = unit;
+ this.value = value;
+ }
+
+ public static Size parse(String providedLongWithUnitString) throws Exception {
+ if (providedLongWithUnitString.equalsIgnoreCase(UNKNOWN)) {
+ return new Size(Unit.NoUnit, UNKNOWN_VALUE);
+ }
+ if (providedLongWithUnitString.equalsIgnoreCase(UNLIMITED)) {
+ return new Size(Unit.NoUnit, UNLIMITED_VALUE);
+ }
+ char lastChar = providedLongWithUnitString.charAt(providedLongWithUnitString.length() - 1);
+ Unit unit = getUnit(lastChar);
+ String argWithoutUnit = removeLastCharIfNeeded(providedLongWithUnitString, unit);
+ return new Size(unit, Long.parseLong(argWithoutUnit));
+ }
+
+ public Unit getUnit() {
+ return unit;
+ }
+
+ public Long getValue() {
+ return value;
+ }
+
+ public long asBytes() {
+ switch (unit) {
+ case G:
+ return value * LongMath.pow(base, 3);
+ case M:
+ return value * LongMath.pow(base, 2);
+ case K:
+ return value * LongMath.pow(base, 1);
+ default:
+ return value;
+ }
+ }
+
+ private static String removeLastCharIfNeeded(String providedLongWithUnitString, Unit unit) {
+ if (unit != Unit.NoUnit) {
+ return providedLongWithUnitString.substring(0, providedLongWithUnitString.length() - 1);
+ } else {
+ return providedLongWithUnitString;
+ }
+ }
+
+ private static Unit getUnit(char lastChar) throws Exception {
+ switch (lastChar) {
+ case 'K' :
+ case 'k' :
+ return Unit.K;
+ case 'M' :
+ case 'm' :
+ return Unit.M;
+ case 'G' :
+ case 'g' :
+ return Unit.G;
+ case 'b' :
+ case 'B' :
+ return Unit.B;
+ case '1' :
+ case '2' :
+ case '3' :
+ case '4' :
+ case '5' :
+ case '6' :
+ case '7' :
+ case '8' :
+ case '9' :
+ case '0' :
+ return Unit.NoUnit;
+ default:
+ throw new Exception("No unit corresponding to char : " + lastChar);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/e3751bea/server/container/util/src/test/java/org/apache/james/util/SizeTest.java
----------------------------------------------------------------------
diff --git a/server/container/util/src/test/java/org/apache/james/util/SizeTest.java b/server/container/util/src/test/java/org/apache/james/util/SizeTest.java
new file mode 100644
index 0000000..0aaeae2
--- /dev/null
+++ b/server/container/util/src/test/java/org/apache/james/util/SizeTest.java
@@ -0,0 +1,73 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.util;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.Test;
+
+public class SizeTest {
+
+ @Test
+ public void testNoUnit() throws Exception {
+ assertThat(Size.parse("1024").asBytes()).isEqualTo(1024);
+ }
+
+ @Test
+ public void testUnitB() throws Exception {
+ assertThat(Size.parse("1024B").asBytes()).isEqualTo(1024);
+ }
+
+ @Test
+ public void testUnitK() throws Exception {
+ assertThat(Size.parse("5K").asBytes()).isEqualTo(5 * 1024);
+ }
+
+ @Test
+ public void testUnitM() throws Exception {
+ assertThat(Size.parse("5M").asBytes()).isEqualTo(5 * 1024 * 1024);
+ }
+
+ @Test
+ public void testUnitG() throws Exception {
+ assertThat(Size.parse("1G").asBytes()).isEqualTo(1024 * 1024 * 1024);
+ }
+
+ @Test
+ public void testUnknown() throws Exception {
+ assertThat(Size.parse("unknown").asBytes()).isEqualTo(Size.UNKNOWN_VALUE);
+ }
+
+ @Test
+ public void testUnlimited() throws Exception {
+ assertThat(Size.parse("unlimited").asBytes()).isEqualTo(Size.UNLIMITED_VALUE);
+ }
+
+ @Test(expected = Exception.class)
+ public void testBadUnit() throws Exception {
+ Size.parse("42T");
+ }
+
+ @Test(expected = NumberFormatException.class)
+ public void testWrongNumber() throws Exception {
+ Size.parse("42RG");
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[12/14] james-project git commit: MAILBOX-338 Test cache eviction
policy
Posted by bt...@apache.org.
MAILBOX-338 Test cache eviction policy
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/69836e43
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/69836e43
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/69836e43
Branch: refs/heads/master
Commit: 69836e436baa2d92988c2b0aef0460b2f49cf7e6
Parents: ed4166a
Author: benwa <bt...@linagora.com>
Authored: Tue May 29 15:27:45 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../mailbox/tika/CachingTextExtractor.java | 7 +-
.../mailbox/tika/CachingTextExtractorTest.java | 105 ++++++++++++++++---
2 files changed, 98 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/69836e43/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
index cfc731b..4429664 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -33,13 +33,13 @@ import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.metrics.api.Metric;
import org.apache.james.metrics.api.MetricFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.Weigher;
import com.google.common.util.concurrent.UncheckedExecutionException;
-
public class CachingTextExtractor implements TextExtractor {
private final TextExtractor underlying;
private final Cache<String, ParsedContent> cache;
@@ -126,4 +126,9 @@ public class CachingTextExtractor implements TextExtractor {
.map(throwable -> (Exception) throwable)
.orElse(e);
}
+
+ @VisibleForTesting
+ long size() {
+ return cache.size();
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/69836e43/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
index e34f95c..519be56 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -19,19 +19,24 @@
package org.apache.james.mailbox.tika;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.util.function.Function;
import java.util.function.Supplier;
+import java.util.stream.IntStream;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
@@ -39,22 +44,30 @@ import org.apache.james.metrics.api.NoopMetricFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import com.github.fge.lambdas.Throwing;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
public class CachingTextExtractorTest {
- public static final ParsedContent RESULT = new ParsedContent("content", ImmutableMap.of());
- public static final Supplier<InputStream> INPUT_STREAM_1 = () -> new ByteArrayInputStream("content1".getBytes(StandardCharsets.UTF_8));
+ private static final ParsedContent RESULT = new ParsedContent("content", ImmutableMap.of());
+ public static final String BIG_STRING = Strings.repeat("0123456789", 103 * 1024);
+ private static final ParsedContent _2MiB_RESULT = new ParsedContent(BIG_STRING, ImmutableMap.of());
+ private static final Function<Integer, InputStream> STREAM_GENERATOR =
+ i -> new ByteArrayInputStream(String.format("content%d", i).getBytes(StandardCharsets.UTF_8));
+ private static final Supplier<InputStream> INPUT_STREAM = () -> STREAM_GENERATOR.apply(1);
+ private static final long CACHE_LIMIT_10_MiB = 10 * 1024 * 1024;
+ private static final String CONTENT_TYPE = "application/bytes";
private CachingTextExtractor textExtractor;
private TextExtractor wrappedTextExtractor;
@BeforeEach
- public void setUp() throws Exception {
+ void setUp() throws Exception {
wrappedTextExtractor = mock(TextExtractor.class);
textExtractor = new CachingTextExtractor(wrappedTextExtractor,
TikaConfiguration.DEFAULT_CACHE_EVICTION_PERIOD,
- TikaConfiguration.DEFAULT_CACHE_LIMIT_100_MB,
+ CACHE_LIMIT_10_MiB,
new NoopMetricFactory());
when(wrappedTextExtractor.extractContent(any(), any()))
@@ -62,40 +75,106 @@ public class CachingTextExtractorTest {
}
@Test
- public void extractContentShouldCallUnderlyingTextExtractor() throws Exception {
- textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes");
+ void extractContentShouldCallUnderlyingTextExtractor() throws Exception {
+ textExtractor.extractContent(INPUT_STREAM.get(), CONTENT_TYPE);
verify(wrappedTextExtractor, times(1)).extractContent(any(), any());
verifyNoMoreInteractions(wrappedTextExtractor);
}
@Test
- public void extractContentShouldAvoidCallingUnderlyingTextExtractorWhenPossible() throws Exception {
- textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes");
- textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes");
+ void extractContentShouldAvoidCallingUnderlyingTextExtractorWhenPossible() throws Exception {
+ textExtractor.extractContent(INPUT_STREAM.get(), CONTENT_TYPE);
+ textExtractor.extractContent(INPUT_STREAM.get(), CONTENT_TYPE);
verify(wrappedTextExtractor, times(1)).extractContent(any(), any());
verifyNoMoreInteractions(wrappedTextExtractor);
}
@Test
- public void extractContentShouldPropagateCheckedException() throws Exception {
+ void extractContentShouldPropagateCheckedException() throws Exception {
IOException ioException = new IOException("Any");
when(wrappedTextExtractor.extractContent(any(), any()))
.thenThrow(ioException);
- assertThatThrownBy(() -> textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes"))
+ assertThatThrownBy(() -> textExtractor.extractContent(INPUT_STREAM.get(), CONTENT_TYPE))
.isEqualTo(ioException);
}
@Test
- public void extractContentShouldPropagateRuntimeException() throws Exception {
+ void extractContentShouldPropagateRuntimeException() throws Exception {
RuntimeException runtimeException = new RuntimeException("Any");
when(wrappedTextExtractor.extractContent(any(), any()))
.thenThrow(runtimeException);
- assertThatThrownBy(() -> textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes"))
+ assertThatThrownBy(() -> textExtractor.extractContent(INPUT_STREAM.get(), CONTENT_TYPE))
.isEqualTo(runtimeException);
}
+ @Test
+ void cacheShouldEvictEntriesWhenFull() throws Exception {
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(_2MiB_RESULT);
+
+ IntStream.range(0, 10)
+ .mapToObj(STREAM_GENERATOR::apply)
+ .forEach(Throwing.consumer(inputStream -> textExtractor.extractContent(inputStream, CONTENT_TYPE)));
+
+ assertThat(textExtractor.size())
+ .isLessThanOrEqualTo(5);
+ }
+
+ @Test
+ void olderEntriesShouldBeEvictedFirst() throws Exception {
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(_2MiB_RESULT);
+
+ IntStream.range(0, 10)
+ .mapToObj(STREAM_GENERATOR::apply)
+ .forEach(Throwing.consumer(inputStream -> textExtractor.extractContent(inputStream, CONTENT_TYPE)));
+
+ reset(wrappedTextExtractor);
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(_2MiB_RESULT);
+
+ textExtractor.extractContent(STREAM_GENERATOR.apply(1), CONTENT_TYPE);
+
+ verify(wrappedTextExtractor).extractContent(any(), any());
+ }
+
+ @Test
+ void youngerEntriesShouldBePreservedByEviction() throws Exception {
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(_2MiB_RESULT);
+
+ IntStream.range(0, 10)
+ .mapToObj(STREAM_GENERATOR::apply)
+ .forEach(Throwing.consumer(inputStream -> textExtractor.extractContent(inputStream, CONTENT_TYPE)));
+
+ reset(wrappedTextExtractor);
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(_2MiB_RESULT);
+
+ textExtractor.extractContent(STREAM_GENERATOR.apply(9), CONTENT_TYPE);
+
+ verifyZeroInteractions(wrappedTextExtractor);
+ }
+
+ @Test
+ void frequentlyAccessedEntriesShouldBePreservedByEviction() throws Exception {
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(_2MiB_RESULT);
+
+ IntStream.range(0, 10)
+ .mapToObj(STREAM_GENERATOR::apply)
+ .peek(Throwing.consumer(any -> textExtractor.extractContent(STREAM_GENERATOR.apply(0), CONTENT_TYPE)))
+ .forEach(Throwing.consumer(inputStream -> textExtractor.extractContent(inputStream, CONTENT_TYPE)));
+
+ reset(wrappedTextExtractor);
+
+ textExtractor.extractContent(STREAM_GENERATOR.apply(0), CONTENT_TYPE);
+
+ verifyZeroInteractions(wrappedTextExtractor);
+ }
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[08/14] james-project git commit: MAILBOX-338 Allow Gauge usage
Posted by bt...@apache.org.
MAILBOX-338 Allow Gauge usage
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/d0fd6648
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/d0fd6648
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/d0fd6648
Branch: refs/heads/master
Commit: d0fd6648992654d4262d6e246d8f37fdce57a5ad
Parents: e3751be
Author: benwa <bt...@linagora.com>
Authored: Sun May 27 11:12:02 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../org/apache/james/metrics/api/Gauge.java | 25 ++++++++++++
.../apache/james/metrics/api/GaugeRegistry.java | 24 +++++++++++
.../james/metrics/api/NoopGaugeRegistry.java | 27 +++++++++++++
.../james/metrics/api/NoopMetricFactory.java | 1 +
.../dropwizard/DropWizardGaugeRegistry.java | 42 ++++++++++++++++++++
.../metrics/logger/DefaultMetricFactory.java | 5 +++
.../modules/server/DropWizardMetricsModule.java | 5 +++
7 files changed, 129 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Gauge.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Gauge.java b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Gauge.java
new file mode 100644
index 0000000..cd524c2
--- /dev/null
+++ b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/Gauge.java
@@ -0,0 +1,25 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.metrics.api;
+
+import java.util.function.Supplier;
+
+public interface Gauge<T> extends Supplier<T> {
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/GaugeRegistry.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/GaugeRegistry.java b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/GaugeRegistry.java
new file mode 100644
index 0000000..d350627
--- /dev/null
+++ b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/GaugeRegistry.java
@@ -0,0 +1,24 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.metrics.api;
+
+public interface GaugeRegistry {
+ <T> GaugeRegistry register(String name, Gauge<T> gauge);
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopGaugeRegistry.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopGaugeRegistry.java b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopGaugeRegistry.java
new file mode 100644
index 0000000..80fe43b
--- /dev/null
+++ b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopGaugeRegistry.java
@@ -0,0 +1,27 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.metrics.api;
+
+public class NoopGaugeRegistry implements GaugeRegistry {
+ @Override
+ public <T> GaugeRegistry register(String name, Gauge<T> gauge) {
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
index 5afacd4..f421075 100644
--- a/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
+++ b/metrics/metrics-api/src/main/java/org/apache/james/metrics/api/NoopMetricFactory.java
@@ -62,4 +62,5 @@ public class NoopMetricFactory implements MetricFactory {
public <T> T withMetric(String name, Supplier<T> operation) {
return operation.get();
}
+
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardGaugeRegistry.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardGaugeRegistry.java b/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardGaugeRegistry.java
new file mode 100644
index 0000000..eaf944e
--- /dev/null
+++ b/metrics/metrics-dropwizard/src/main/java/org/apache/james/metrics/dropwizard/DropWizardGaugeRegistry.java
@@ -0,0 +1,42 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.metrics.dropwizard;
+
+import javax.inject.Inject;
+
+import org.apache.james.metrics.api.Gauge;
+import org.apache.james.metrics.api.GaugeRegistry;
+
+import com.codahale.metrics.MetricRegistry;
+
+public class DropWizardGaugeRegistry implements GaugeRegistry {
+ private final MetricRegistry metricRegistry;
+
+ @Inject
+ public DropWizardGaugeRegistry(MetricRegistry metricRegistry) {
+ this.metricRegistry = metricRegistry;
+ }
+
+ @Override
+ public <T> GaugeRegistry register(String name, Gauge<T> gauge) {
+ metricRegistry.gauge(name, () -> gauge::get);
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java b/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
index 2997805..f7f5bb4 100644
--- a/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
+++ b/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
@@ -20,6 +20,7 @@ package org.apache.james.metrics.logger;
import java.util.function.Supplier;
+import org.apache.james.metrics.api.Gauge;
import org.apache.james.metrics.api.Metric;
import org.apache.james.metrics.api.MetricFactory;
import org.apache.james.metrics.api.TimeMetric;
@@ -50,4 +51,8 @@ public class DefaultMetricFactory implements MetricFactory {
}
}
+ @Override
+ public <T> void register(String name, Gauge<T> gauge) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/d0fd6648/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java b/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
index 2e5d3c9..c5eb73d 100644
--- a/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
+++ b/server/container/guice/guice-common/src/main/java/org/apache/james/modules/server/DropWizardMetricsModule.java
@@ -24,7 +24,9 @@ import java.util.List;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.HierarchicalConfiguration;
import org.apache.james.lifecycle.api.Configurable;
+import org.apache.james.metrics.api.GaugeRegistry;
import org.apache.james.metrics.api.MetricFactory;
+import org.apache.james.metrics.dropwizard.DropWizardGaugeRegistry;
import org.apache.james.metrics.dropwizard.DropWizardJVMMetrics;
import org.apache.james.metrics.dropwizard.DropWizardMetricFactory;
import org.apache.james.utils.ConfigurationPerformer;
@@ -44,9 +46,12 @@ public class DropWizardMetricsModule extends AbstractModule {
protected void configure() {
bind(MetricRegistry.class).in(Scopes.SINGLETON);
bind(DropWizardMetricFactory.class).in(Scopes.SINGLETON);
+ bind(DropWizardGaugeRegistry.class).in(Scopes.SINGLETON);
bind(DropWizardJVMMetrics.class).in(Scopes.SINGLETON);
bind(MetricFactory.class).to(DropWizardMetricFactory.class);
+ bind(GaugeRegistry.class).to(DropWizardGaugeRegistry.class);
+
Multibinder.newSetBinder(binder(), ConfigurationPerformer.class).addBinding().to(DropWizardConfigurationPerformer.class);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[02/14] james-project git commit: JAMES-2404 Improve
ParametersExtractor
Posted by bt...@apache.org.
JAMES-2404 Improve ParametersExtractor
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/d8ef8aea
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/d8ef8aea
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/d8ef8aea
Branch: refs/heads/master
Commit: d8ef8aea49303630eb03683bd09d7d196a409a1c
Parents: 006c81b
Author: benwa <bt...@linagora.com>
Authored: Wed May 30 09:53:16 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:15 2018 +0700
----------------------------------------------------------------------
.../webadmin/utils/ParametersExtractor.java | 21 ++++++++++----------
1 file changed, 10 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/d8ef8aea/server/protocols/webadmin/webadmin-core/src/main/java/org/apache/james/webadmin/utils/ParametersExtractor.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-core/src/main/java/org/apache/james/webadmin/utils/ParametersExtractor.java b/server/protocols/webadmin/webadmin-core/src/main/java/org/apache/james/webadmin/utils/ParametersExtractor.java
index 2cba692..582f1d2 100644
--- a/server/protocols/webadmin/webadmin-core/src/main/java/org/apache/james/webadmin/utils/ParametersExtractor.java
+++ b/server/protocols/webadmin/webadmin-core/src/main/java/org/apache/james/webadmin/utils/ParametersExtractor.java
@@ -19,6 +19,7 @@
package org.apache.james.webadmin.utils;
import java.util.Optional;
+import java.util.function.Function;
import org.apache.james.util.streams.Limit;
import org.apache.james.util.streams.Offset;
@@ -34,30 +35,28 @@ public class ParametersExtractor {
public static final String OFFSET_PARAMETER_NAME = "offset";
public static Limit extractLimit(Request request) {
- return Limit.from(assertPositiveInteger(request, LIMIT_PARAMETER_NAME)
+ return Limit.from(extractPositiveInteger(request, LIMIT_PARAMETER_NAME)
.map(value -> assertNotZero(value, LIMIT_PARAMETER_NAME)));
}
public static Offset extractOffset(Request request) {
- return Offset.from(assertPositiveInteger(request, OFFSET_PARAMETER_NAME));
+ return Offset.from(extractPositiveInteger(request, OFFSET_PARAMETER_NAME));
}
public static Optional<Double> extractPositiveDouble(Request request, String parameterName) {
- return extractPositiveNumber(request, parameterName)
- .map(Number::doubleValue);
+ return extractPositiveNumber(request, parameterName, Double::valueOf);
}
- private static Optional<Integer> assertPositiveInteger(Request request, String parameterName) {
- return extractPositiveNumber(request, parameterName)
- .map(Number::intValue);
+ public static Optional<Integer> extractPositiveInteger(Request request, String parameterName) {
+ return extractPositiveNumber(request, parameterName, Integer::valueOf);
}
- private static Optional<Number> extractPositiveNumber(Request request, String parameterName) {
+ private static <T extends Number> Optional<T> extractPositiveNumber(Request request, String parameterName, Function<String, T> toNumber) {
try {
return Optional.ofNullable(request.queryParams(parameterName))
.filter(s -> !Strings.isNullOrEmpty(s))
- .map(Double::valueOf)
- .map(value -> assertPositive(value, parameterName));
+ .map(toNumber)
+ .map(value -> ParametersExtractor.assertPositive(value, parameterName));
} catch (NumberFormatException e) {
throw ErrorResponder.builder()
.statusCode(HttpStatus.BAD_REQUEST_400)
@@ -68,7 +67,7 @@ public class ParametersExtractor {
}
}
- private static Number assertPositive(Number value, String parameterName) {
+ private static <T extends Number> T assertPositive(T value, String parameterName) {
if (value.doubleValue() < 0) {
throw ErrorResponder.builder()
.statusCode(HttpStatus.BAD_REQUEST_400)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[05/14] james-project git commit: MAILBOX-338 Allow to disable Tika
Posted by bt...@apache.org.
MAILBOX-338 Allow to disable Tika
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/ed4166a5
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/ed4166a5
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/ed4166a5
Branch: refs/heads/master
Commit: ed4166a52565746b4a183e4db2529738ac5823a9
Parents: 64b2c28
Author: benwa <bt...@linagora.com>
Authored: Sun May 27 13:54:17 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
mailbox/tika/pom.xml | 20 +++-
.../james/mailbox/tika/TikaConfiguration.java | 106 ++++++++++++++++---
.../mailbox/tika/CachingTextExtractorTest.java | 11 +-
.../mailbox/TikaConfigurationReader.java | 30 ++++--
.../modules/mailbox/TikaMailboxModule.java | 31 +++---
.../mailbox/TikaConfigurationReaderTest.java | 87 +++++++++++----
6 files changed, 221 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/ed4166a5/mailbox/tika/pom.xml
----------------------------------------------------------------------
diff --git a/mailbox/tika/pom.xml b/mailbox/tika/pom.xml
index 8d78433..f926a35 100644
--- a/mailbox/tika/pom.xml
+++ b/mailbox/tika/pom.xml
@@ -72,11 +72,6 @@
<artifactId>commons-configuration</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
@@ -91,6 +86,21 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.platform</groupId>
+ <artifactId>junit-platform-launcher</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.vintage</groupId>
+ <artifactId>junit-vintage-engine</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/james-project/blob/ed4166a5/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
index e7994b7..113c1e9 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
@@ -22,19 +22,18 @@ package org.apache.james.mailbox.tika;
import java.time.Duration;
import java.util.Objects;
import java.util.Optional;
+import java.util.concurrent.TimeUnit;
import org.apache.james.util.Port;
import com.google.common.base.Preconditions;
+import com.google.common.primitives.Ints;
public class TikaConfiguration {
- public static Builder builder() {
- return new Builder();
- }
-
public static class Builder {
-
+ private Optional<Boolean> isEnabled;
+ private Optional<Boolean> isCacheEnabled;
private Optional<String> host;
private Optional<Integer> port;
private Optional<Integer> timeoutInMillis;
@@ -42,6 +41,8 @@ public class TikaConfiguration {
private Optional<Long> cacheWeightInBytes;
private Builder() {
+ isEnabled = Optional.empty();
+ isCacheEnabled = Optional.empty();
host = Optional.empty();
port = Optional.empty();
timeoutInMillis = Optional.empty();
@@ -49,8 +50,47 @@ public class TikaConfiguration {
cacheWeightInBytes = Optional.empty();
}
+ public Builder enable(Optional<Boolean> isEnabled) {
+ Preconditions.checkNotNull(isEnabled);
+ this.isEnabled = isEnabled;
+ return this;
+ }
+
+ public Builder enabled() {
+ this.isEnabled = Optional.of(true);
+ return this;
+ }
+
+ public Builder disabled() {
+ this.isEnabled = Optional.of(false);
+ return this;
+ }
+
+ public Builder cacheEnable(Optional<Boolean> isEnabled) {
+ Preconditions.checkNotNull(isEnabled);
+ this.isCacheEnabled = isEnabled;
+ return this;
+ }
+
+ public Builder cacheEnabled() {
+ this.isCacheEnabled = Optional.of(true);
+ return this;
+ }
+
+ public Builder cacheDisabled() {
+ this.isCacheEnabled = Optional.of(false);
+ return this;
+ }
+
public Builder host(String host) {
- this.host = Optional.ofNullable(host);
+ Preconditions.checkNotNull(host);
+ this.host = Optional.of(host);
+ return this;
+ }
+
+ public Builder host(Optional<String> host) {
+ Preconditions.checkNotNull(host);
+ this.host = host;
return this;
}
@@ -59,11 +99,23 @@ public class TikaConfiguration {
return this;
}
+ public Builder port(Optional<Integer> port) {
+ Preconditions.checkNotNull(port);
+ this.port = port;
+ return this;
+ }
+
public Builder timeoutInMillis(int timeoutInMillis) {
this.timeoutInMillis = Optional.of(timeoutInMillis);
return this;
}
+ public Builder timeoutInMillis(Optional<Integer> timeoutInMillis) {
+ Preconditions.checkNotNull(timeoutInMillis);
+ this.timeoutInMillis = timeoutInMillis;
+ return this;
+ }
+
public Builder cacheEvictionPeriod(Duration duration) {
this.cacheEvictionPeriod = Optional.of(duration);
return this;
@@ -85,26 +137,41 @@ public class TikaConfiguration {
}
public TikaConfiguration build() {
- Preconditions.checkState(host.isPresent(), "'host' is mandatory");
- Preconditions.checkState(port.isPresent(), "'port' is mandatory");
- Preconditions.checkState(timeoutInMillis.isPresent(), "'timeoutInMillis' is mandatory");
- Port.assertValid(port.get());
-
- return new TikaConfiguration(host.get(), port.get(), timeoutInMillis.get(),
+ port.ifPresent(Port::assertValid);
+
+ return new TikaConfiguration(
+ isEnabled.orElse(DEFAULT_DISABLED),
+ isCacheEnabled.orElse(DEFAULT_DISABLED),
+ host.orElse(DEFAULT_HOST),
+ port.orElse(DEFAULT_PORT),
+ timeoutInMillis.orElse(DEFAULT_TIMEOUT_IN_MS),
cacheEvictionPeriod.orElse(DEFAULT_CACHE_EVICTION_PERIOD),
cacheWeightInBytes.orElse(DEFAULT_CACHE_LIMIT_100_MB));
}
}
+
public static final long DEFAULT_CACHE_LIMIT_100_MB = 1024L * 1024L * 100L;
public static final Duration DEFAULT_CACHE_EVICTION_PERIOD = Duration.ofDays(1);
+ public static final boolean DEFAULT_DISABLED = false;
+ public static final String DEFAULT_HOST = "127.0.0.1";
+ public static final int DEFAULT_PORT = 9998;
+ public static final int DEFAULT_TIMEOUT_IN_MS = Ints.checkedCast(TimeUnit.SECONDS.toMillis(30));
+
+ public static Builder builder() {
+ return new Builder();
+ }
+ private final boolean enabled;
+ private final boolean cacheEnabled;
private final String host;
private final int port;
private final int timeoutInMillis;
private final Duration cacheEvictionPeriod;
private final long cacheWeightInBytes;
- private TikaConfiguration(String host, int port, int timeoutInMillis, Duration cacheEvictionPeriod, long cacheWeightInBytes) {
+ private TikaConfiguration(boolean enabled, boolean cacheEnabled, String host, int port, int timeoutInMillis, Duration cacheEvictionPeriod, long cacheWeightInBytes) {
+ this.enabled = enabled;
+ this.cacheEnabled = cacheEnabled;
this.host = host;
this.port = port;
this.timeoutInMillis = timeoutInMillis;
@@ -112,6 +179,14 @@ public class TikaConfiguration {
this.cacheWeightInBytes = cacheWeightInBytes;
}
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public boolean isCacheEnabled() {
+ return cacheEnabled;
+ }
+
public String getHost() {
return host;
}
@@ -137,7 +212,8 @@ public class TikaConfiguration {
if (o instanceof TikaConfiguration) {
TikaConfiguration that = (TikaConfiguration) o;
- return Objects.equals(this.port, that.port)
+ return Objects.equals(this.enabled, that.enabled)
+ && Objects.equals(this.port, that.port)
&& Objects.equals(this.timeoutInMillis, that.timeoutInMillis)
&& Objects.equals(this.cacheWeightInBytes, that.cacheWeightInBytes)
&& Objects.equals(this.host, that.host)
@@ -148,7 +224,7 @@ public class TikaConfiguration {
@Override
public final int hashCode() {
- return Objects.hash(host, port, timeoutInMillis, cacheEvictionPeriod, cacheWeightInBytes);
+ return Objects.hash(enabled, host, port, timeoutInMillis, cacheEvictionPeriod, cacheWeightInBytes);
}
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/ed4166a5/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
index 54e743f..e34f95c 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -36,19 +36,20 @@ import java.util.function.Supplier;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
import org.apache.james.metrics.api.NoopMetricFactory;
-import org.junit.Before;
-import org.junit.Test;
-import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import com.google.common.collect.ImmutableMap;
public class CachingTextExtractorTest {
public static final ParsedContent RESULT = new ParsedContent("content", ImmutableMap.of());
public static final Supplier<InputStream> INPUT_STREAM_1 = () -> new ByteArrayInputStream("content1".getBytes(StandardCharsets.UTF_8));
- private TextExtractor textExtractor;
+ private CachingTextExtractor textExtractor;
private TextExtractor wrappedTextExtractor;
- @Before
+ @BeforeEach
public void setUp() throws Exception {
wrappedTextExtractor = mock(TextExtractor.class);
textExtractor = new CachingTextExtractor(wrappedTextExtractor,
http://git-wip-us.apache.org/repos/asf/james-project/blob/ed4166a5/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
index 13ca0cd..6907330 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
@@ -21,7 +21,6 @@ package org.apache.james.modules.mailbox;
import java.time.Duration;
import java.util.Optional;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.james.mailbox.tika.TikaConfiguration;
@@ -29,19 +28,32 @@ import org.apache.james.util.Size;
import org.apache.james.util.TimeConverter;
import com.github.fge.lambdas.Throwing;
-import com.google.common.primitives.Ints;
public class TikaConfigurationReader {
+ public static final String TIKA_ENABLED = "tika.enabled";
+ public static final String TIKA_CACHE_ENABLED = "tika.cache.enabled";
public static final String TIKA_HOST = "tika.host";
public static final String TIKA_PORT = "tika.port";
public static final String TIKA_TIMEOUT_IN_MS = "tika.timeoutInMillis";
- public static final String DEFAULT_HOST = "127.0.0.1";
- public static final int DEFAULT_PORT = 9998;
public static final String TIKA_CACHE_EVICTION_PERIOD = "tika.cache.eviction.period";
public static final String TIKA_CACHE_WEIGHT_MAX = "tika.cache.weight.max";
- public static final int DEFAULT_TIMEOUT_IN_MS = Ints.checkedCast(TimeUnit.SECONDS.toMillis(30));
public static TikaConfiguration readTikaConfiguration(PropertiesConfiguration configuration) {
+ Optional<Boolean> enabled = Optional.ofNullable(
+ configuration.getBoolean(TIKA_ENABLED, null));
+
+ Optional<Boolean> cacheEnabled = Optional.ofNullable(
+ configuration.getBoolean(TIKA_CACHE_ENABLED, null));
+
+ Optional<String> host = Optional.ofNullable(
+ configuration.getString(TIKA_HOST, null));
+
+ Optional<Integer> port = Optional.ofNullable(
+ configuration.getInteger(TIKA_PORT, null));
+
+ Optional<Integer> timeoutInMillis = Optional.ofNullable(
+ configuration.getInteger(TIKA_TIMEOUT_IN_MS, null));
+
Optional<Duration> cacheEvictionPeriod = Optional.ofNullable(
configuration.getString(TIKA_CACHE_EVICTION_PERIOD,
null))
@@ -54,9 +66,11 @@ public class TikaConfigurationReader {
.map(Size::asBytes);
return TikaConfiguration.builder()
- .host(configuration.getString(TIKA_HOST, DEFAULT_HOST))
- .port(configuration.getInt(TIKA_PORT, DEFAULT_PORT))
- .timeoutInMillis(configuration.getInt(TIKA_TIMEOUT_IN_MS, DEFAULT_TIMEOUT_IN_MS))
+ .enable(enabled)
+ .host(host)
+ .port(port)
+ .timeoutInMillis(timeoutInMillis)
+ .cacheEnable(cacheEnabled)
.cacheEvictionPeriod(cacheEvictionPeriod)
.cacheWeightInBytes(cacheWeight)
.build();
http://git-wip-us.apache.org/repos/asf/james-project/blob/ed4166a5/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
index b5421f6..08bcfa5 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
@@ -19,10 +19,6 @@
package org.apache.james.modules.mailbox;
-import static org.apache.james.modules.mailbox.TikaConfigurationReader.DEFAULT_HOST;
-import static org.apache.james.modules.mailbox.TikaConfigurationReader.DEFAULT_PORT;
-import static org.apache.james.modules.mailbox.TikaConfigurationReader.DEFAULT_TIMEOUT_IN_MS;
-
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
@@ -31,6 +27,7 @@ import javax.inject.Singleton;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.store.extractor.DefaultTextExtractor;
import org.apache.james.mailbox.tika.CachingTextExtractor;
import org.apache.james.mailbox.tika.TikaConfiguration;
import org.apache.james.mailbox.tika.TikaHttpClient;
@@ -71,22 +68,30 @@ public class TikaMailboxModule extends AbstractModule {
return TikaConfigurationReader.readTikaConfiguration(configuration);
} catch (FileNotFoundException e) {
- LOGGER.warn("Could not find {} configuration file. Using {}:{} as contact point", TIKA_CONFIGURATION_NAME, DEFAULT_HOST, DEFAULT_PORT);
+ LOGGER.warn("Could not find {} configuration file. Disabling Tika.", TIKA_CONFIGURATION_NAME);
return TikaConfiguration.builder()
- .host(DEFAULT_HOST)
- .port(DEFAULT_PORT)
- .timeoutInMillis(DEFAULT_TIMEOUT_IN_MS)
- .build();
+ .disabled()
+ .build();
}
}
@Provides
@Singleton
private TextExtractor provideTextExtractor(TikaTextExtractor textExtractor, TikaConfiguration configuration, MetricFactory metricFactory) {
- return new CachingTextExtractor(
- textExtractor,
- configuration.getCacheEvictionPeriod(),
- configuration.getCacheWeightInBytes(), metricFactory);
+ if (configuration.isEnabled() && configuration.isCacheEnabled()) {
+ LOGGER.info("Tika cache has been enabled.");
+ return new CachingTextExtractor(
+ textExtractor,
+ configuration.getCacheEvictionPeriod(),
+ configuration.getCacheWeightInBytes(), metricFactory);
+ }
+ if (configuration.isEnabled()) {
+ return textExtractor;
+ }
+ LOGGER.info("Tika text extraction has been disabled." +
+ " Using DefaultTextExtractor instead. " +
+ "No complex extraction will be done.");
+ return new DefaultTextExtractor();
}
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/ed4166a5/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
index 484c8f0..c66df2d 100644
--- a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
+++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
@@ -34,16 +34,19 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldAcceptMandatoryValues() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.port=889\n" +
- "tika.timeoutInMillis=500\n"));
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("172.0.0.5")
.port(889)
.timeoutInMillis(500)
+ .cacheDisabled()
.cacheWeightInBytes(100L * 1024L *1024L)
.cacheEvictionPeriod(Duration.ofDays(1))
.build());
@@ -53,12 +56,14 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldReturnDefaultOnMissingHost() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
- "tika.port=889\n" +
- "tika.timeoutInMillis=500\n"));
+ "tika.enabled=true\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("127.0.0.1")
.port(889)
.timeoutInMillis(500)
@@ -69,12 +74,14 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldReturnDefaultOnMissingPort() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.timeoutInMillis=500\n"));
+ "tika.timeoutInMillis=500\n"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("172.0.0.5")
.port(9998)
.timeoutInMillis(500)
@@ -85,31 +92,46 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldReturnDefaultOnMissingTimeout() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.port=889\n"));
+ "tika.port=889\n"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("172.0.0.5")
.port(889)
.timeoutInMillis(30 * 1000)
.build());
}
+ @Test
+ public void tikaShouldBeDisabledByDefault() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(""));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .disabled()
+ .build());
+ }
@Test
public void readTikaConfigurationShouldParseUnitForCacheEvictionPeriod() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.port=889\n" +
- "tika.timeoutInMillis=500\n" +
- "tika.cache.eviction.period=2H"));
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.eviction.period=2H"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("172.0.0.5")
.port(889)
.timeoutInMillis(500)
@@ -121,14 +143,16 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldDefaultToSecondWhenMissingUnitForCacheEvitionPeriod() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.port=889\n" +
- "tika.timeoutInMillis=500\n" +
- "tika.cache.eviction.period=3600"));
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.eviction.period=3600"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("172.0.0.5")
.port(889)
.timeoutInMillis(500)
@@ -140,14 +164,16 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldParseUnitForCacheWeightMax() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.port=889\n" +
- "tika.timeoutInMillis=500\n" +
- "tika.cache.weight.max=200M"));
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.weight.max=200M"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
.host("172.0.0.5")
.port(889)
.timeoutInMillis(500)
@@ -159,14 +185,39 @@ public class TikaConfigurationReaderTest {
public void readTikaConfigurationShouldDefaultToByteAsSizeUnit() throws Exception {
PropertiesConfiguration configuration = new PropertiesConfiguration();
configuration.load(new StringReader(
+ "tika.enabled=true\n" +
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.weight.max=1520000"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .enabled()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(500)
+ .cacheWeightInBytes(1520000)
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldEnableCacheWhenConfigured() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.enabled=true\n" +
+ "tika.cache.enabled=true\n" +
"tika.host=172.0.0.5\n" +
- "tika.port=889\n" +
- "tika.timeoutInMillis=500\n" +
- "tika.cache.weight.max=1520000"));
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.weight.max=1520000"));
assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
.isEqualTo(
TikaConfiguration.builder()
+ .enabled()
+ .cacheEnabled()
.host("172.0.0.5")
.port(889)
.timeoutInMillis(500)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[04/14] james-project git commit: MAILBOX-338 Metrics for
TextExtractor cache
Posted by bt...@apache.org.
MAILBOX-338 Metrics for TextExtractor cache
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/64b2c285
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/64b2c285
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/64b2c285
Branch: refs/heads/master
Commit: 64b2c285fdaa710e34b4c0e58343d92bcf9ffee4
Parents: 6d3396e
Author: benwa <bt...@linagora.com>
Authored: Sun May 27 11:32:06 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../mailbox/tika/CachingTextExtractor.java | 49 +++++++++++++++++++-
.../mailbox/tika/CachingTextExtractorTest.java | 4 +-
.../modules/mailbox/TikaMailboxModule.java | 5 +-
3 files changed, 53 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/64b2c285/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
index 8bfed1b..cfc731b 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -30,9 +30,12 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.metrics.api.Metric;
+import org.apache.james.metrics.api.MetricFactory;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
import com.google.common.cache.Weigher;
import com.google.common.util.concurrent.UncheckedExecutionException;
@@ -40,16 +43,58 @@ import com.google.common.util.concurrent.UncheckedExecutionException;
public class CachingTextExtractor implements TextExtractor {
private final TextExtractor underlying;
private final Cache<String, ParsedContent> cache;
+ private final Metric weightMetric;
- public CachingTextExtractor(TextExtractor underlying, Duration cacheEvictionPeriod, Long cacheWeightInBytes) {
+ public CachingTextExtractor(TextExtractor underlying, Duration cacheEvictionPeriod, Long cacheWeightInBytes, MetricFactory metricFactory) {
this.underlying = underlying;
+ this.weightMetric = metricFactory.generate("textExtractor.cache.weight");
+
+ Weigher<String, ParsedContent> weigher =
+ (key, parsedContent) -> {
+ int size = getSize(parsedContent);
+ weightMetric.add(size);
+ return size;
+ };
+ RemovalListener<String, ParsedContent> removalListener =
+ notification -> Optional.ofNullable(notification.getValue())
+ .map(this::getSize)
+ .ifPresent(weightMetric::remove);
- Weigher<String, ParsedContent> weigher = (key, parsedContent) -> getSize(parsedContent);
this.cache = CacheBuilder.<String, String>newBuilder()
.expireAfterAccess(cacheEvictionPeriod.toMillis(), TimeUnit.MILLISECONDS)
.maximumWeight(cacheWeightInBytes)
.weigher(weigher)
+ .recordStats()
+ .removalListener(removalListener)
.build();
+ recordStats(metricFactory);
+ }
+
+ public void recordStats(MetricFactory metricFactory) {
+ metricFactory.register(
+ "textExtractor.cache.hit.rate",
+ () -> cache.stats().hitRate());
+ metricFactory.register(
+ "textExtractor.cache.hit.count",
+ () -> cache.stats().hitCount());
+ metricFactory.register(
+ "textExtractor.cache.load.count",
+ () -> cache.stats().loadCount());
+ metricFactory.register(
+ "textExtractor.cache.eviction.count",
+ () -> cache.stats().evictionCount());
+ metricFactory.register(
+ "textExtractor.cache.load.exception.rate",
+ () -> cache.stats().loadExceptionRate());
+ metricFactory.register(
+ "textExtractor.cache.load.miss.rate",
+ () -> cache.stats().missRate());
+ metricFactory.register(
+ "textExtractor.cache.load.miss.count",
+ () -> cache.stats().missCount());
+ metricFactory.register(
+ "textExtractor.cache.size",
+ cache::size);
}
private int getSize(ParsedContent parsedContent) {
http://git-wip-us.apache.org/repos/asf/james-project/blob/64b2c285/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
index 154a5ff..54e743f 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -35,6 +35,7 @@ import java.util.function.Supplier;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.metrics.api.NoopMetricFactory;
import org.junit.Before;
import org.junit.Test;
import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
@@ -52,7 +53,8 @@ public class CachingTextExtractorTest {
wrappedTextExtractor = mock(TextExtractor.class);
textExtractor = new CachingTextExtractor(wrappedTextExtractor,
TikaConfiguration.DEFAULT_CACHE_EVICTION_PERIOD,
- TikaConfiguration.DEFAULT_CACHE_LIMIT_100_MB);
+ TikaConfiguration.DEFAULT_CACHE_LIMIT_100_MB,
+ new NoopMetricFactory());
when(wrappedTextExtractor.extractContent(any(), any()))
.thenReturn(RESULT);
http://git-wip-us.apache.org/repos/asf/james-project/blob/64b2c285/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
index 6fcd505..b5421f6 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
@@ -36,6 +36,7 @@ import org.apache.james.mailbox.tika.TikaConfiguration;
import org.apache.james.mailbox.tika.TikaHttpClient;
import org.apache.james.mailbox.tika.TikaHttpClientImpl;
import org.apache.james.mailbox.tika.TikaTextExtractor;
+import org.apache.james.metrics.api.MetricFactory;
import org.apache.james.utils.PropertiesProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -81,11 +82,11 @@ public class TikaMailboxModule extends AbstractModule {
@Provides
@Singleton
- private TextExtractor provideTextExtractor(TikaTextExtractor textExtractor, TikaConfiguration configuration) {
+ private TextExtractor provideTextExtractor(TikaTextExtractor textExtractor, TikaConfiguration configuration, MetricFactory metricFactory) {
return new CachingTextExtractor(
textExtractor,
configuration.getCacheEvictionPeriod(),
- configuration.getCacheWeightInBytes());
+ configuration.getCacheWeightInBytes(), metricFactory);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[06/14] james-project git commit: JAMES-2393 PLUGINS modules were not
registered for Cassandra LDAP
Posted by bt...@apache.org.
JAMES-2393 PLUGINS modules were not registered for Cassandra LDAP
Structurally solve this by exposing the right module aggregate (like for memory)
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/905e9425
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/905e9425
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/905e9425
Branch: refs/heads/master
Commit: 905e942507f9b02d88e9d7d106b666af4bd552f3
Parents: f8faf50
Author: benwa <bt...@linagora.com>
Authored: Wed May 30 11:15:52 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../james/mpt/smtp/host/CassandraJamesSmtpHostSystem.java | 2 +-
.../main/java/org/apache/james/CassandraJamesServerMain.java | 7 ++++++-
.../src/test/java/org/apache/james/CassandraJmapTestRule.java | 6 +++---
.../java/org/apache/james/JamesCapabilitiesServerTest.java | 3 ++-
.../java/org/apache/james/CassandraLdapJamesServerMain.java | 4 +++-
.../apache/james/jmap/cassandra/CassandraJmapExtension.java | 5 +++--
.../james/jmap/cassandra/cucumber/CassandraStepdefs.java | 5 +++--
7 files changed, 21 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/mpt/impl/smtp/cassandra/src/test/java/org/apache/james/mpt/smtp/host/CassandraJamesSmtpHostSystem.java
----------------------------------------------------------------------
diff --git a/mpt/impl/smtp/cassandra/src/test/java/org/apache/james/mpt/smtp/host/CassandraJamesSmtpHostSystem.java b/mpt/impl/smtp/cassandra/src/test/java/org/apache/james/mpt/smtp/host/CassandraJamesSmtpHostSystem.java
index ac10a8f..26a68fc 100644
--- a/mpt/impl/smtp/cassandra/src/test/java/org/apache/james/mpt/smtp/host/CassandraJamesSmtpHostSystem.java
+++ b/mpt/impl/smtp/cassandra/src/test/java/org/apache/james/mpt/smtp/host/CassandraJamesSmtpHostSystem.java
@@ -91,7 +91,7 @@ public class CassandraJamesSmtpHostSystem extends ExternalSessionFactory impleme
}
@Override
- public void afterTest() throws Exception {
+ public void afterTest() {
jamesServer.stop();
embeddedElasticSearch.after();
folder.delete();
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
index cc023c1..eac26ac 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/CassandraJamesServerMain.java
@@ -98,10 +98,15 @@ public class CassandraJamesServerMain {
new TikaMailboxModule(),
new SpamAssassinListenerModule());
+ public static Module ALL_BUT_JMX_CASSANDRA_MODULE = Modules.combine(
+ CASSANDRA_SERVER_MODULE,
+ PROTOCOLS,
+ PLUGINS);
+
public static void main(String[] args) throws Exception {
Configuration configuration = Configuration.builder().useWorkingDirectoryEnvProperty().build();
GuiceJamesServer server = new GuiceJamesServer(configuration)
- .combineWith(CASSANDRA_SERVER_MODULE, PROTOCOLS, PLUGINS, new JMXServerModule());
+ .combineWith(ALL_BUT_JMX_CASSANDRA_MODULE, new JMXServerModule());
server.start();
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/server/container/guice/cassandra-guice/src/test/java/org/apache/james/CassandraJmapTestRule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/CassandraJmapTestRule.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/CassandraJmapTestRule.java
index ef5585b..13c49e3 100644
--- a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/CassandraJmapTestRule.java
+++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/CassandraJmapTestRule.java
@@ -19,6 +19,8 @@
package org.apache.james;
+import static org.apache.james.CassandraJamesServerMain.ALL_BUT_JMX_CASSANDRA_MODULE;
+
import java.io.IOException;
import org.apache.james.mailbox.extractor.TextExtractor;
@@ -60,9 +62,7 @@ public class CassandraJmapTestRule implements TestRule {
.build();
return new GuiceJamesServer(configuration)
- .combineWith(CassandraJamesServerMain.CASSANDRA_SERVER_MODULE,
- CassandraJamesServerMain.PROTOCOLS,
- CassandraJamesServerMain.PLUGINS)
+ .combineWith(ALL_BUT_JMX_CASSANDRA_MODULE)
.overrideWith(binder -> binder.bind(TextExtractor.class).to(PDFTextExtractor.class))
.overrideWith(new TestJMAPServerModule(LIMIT_TO_10_MESSAGES))
.overrideWith(new TestESMetricReporterModule())
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/server/container/guice/cassandra-guice/src/test/java/org/apache/james/JamesCapabilitiesServerTest.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/JamesCapabilitiesServerTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/JamesCapabilitiesServerTest.java
index fd1173b..80132c1 100644
--- a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/JamesCapabilitiesServerTest.java
+++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/JamesCapabilitiesServerTest.java
@@ -18,6 +18,7 @@
****************************************************************/
package org.apache.james;
+import static org.apache.james.CassandraJamesServerMain.ALL_BUT_JMX_CASSANDRA_MODULE;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -66,7 +67,7 @@ public class JamesCapabilitiesServerTest {
.build();
return new GuiceJamesServer(configuration)
- .combineWith(CassandraJamesServerMain.CASSANDRA_SERVER_MODULE, CassandraJamesServerMain.PROTOCOLS)
+ .combineWith(ALL_BUT_JMX_CASSANDRA_MODULE)
.overrideWith((binder) -> binder.bind(PersistenceAdapter.class).to(MemoryPersistenceAdapter.class))
.overrideWith(new TestElasticSearchModule(embeddedElasticSearch),
cassandraServer.getModule(),
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/server/container/guice/cassandra-ldap-guice/src/main/java/org/apache/james/CassandraLdapJamesServerMain.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-ldap-guice/src/main/java/org/apache/james/CassandraLdapJamesServerMain.java b/server/container/guice/cassandra-ldap-guice/src/main/java/org/apache/james/CassandraLdapJamesServerMain.java
index 61230f8..488ac6f 100644
--- a/server/container/guice/cassandra-ldap-guice/src/main/java/org/apache/james/CassandraLdapJamesServerMain.java
+++ b/server/container/guice/cassandra-ldap-guice/src/main/java/org/apache/james/CassandraLdapJamesServerMain.java
@@ -19,6 +19,8 @@
package org.apache.james;
+import static org.apache.james.CassandraJamesServerMain.ALL_BUT_JMX_CASSANDRA_MODULE;
+
import org.apache.james.data.LdapUsersRepositoryModule;
import org.apache.james.modules.server.JMXServerModule;
import org.apache.james.server.core.configuration.Configuration;
@@ -28,7 +30,7 @@ import com.google.inject.util.Modules;
public class CassandraLdapJamesServerMain {
- public static final Module cassandraLdapServerModule = Modules.override(CassandraJamesServerMain.CASSANDRA_SERVER_MODULE, CassandraJamesServerMain.PROTOCOLS)
+ public static final Module cassandraLdapServerModule = Modules.override(ALL_BUT_JMX_CASSANDRA_MODULE)
.with(new LdapUsersRepositoryModule());
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/CassandraJmapExtension.java
----------------------------------------------------------------------
diff --git a/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/CassandraJmapExtension.java b/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/CassandraJmapExtension.java
index 223460d..5a0dd48 100644
--- a/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/CassandraJmapExtension.java
+++ b/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/CassandraJmapExtension.java
@@ -18,9 +18,10 @@
****************************************************************/
package org.apache.james.jmap.cassandra;
+import static org.apache.james.CassandraJamesServerMain.ALL_BUT_JMX_CASSANDRA_MODULE;
+
import java.io.IOException;
-import org.apache.james.CassandraJamesServerMain;
import org.apache.james.DockerCassandraRule;
import org.apache.james.GuiceJamesServer;
import org.apache.james.backends.es.EmbeddedElasticSearch;
@@ -69,7 +70,7 @@ public class CassandraJmapExtension implements BeforeAllCallback, AfterAllCallba
.build();
return new JamesWithSpamAssassin(
new GuiceJamesServer(configuration)
- .combineWith(CassandraJamesServerMain.CASSANDRA_SERVER_MODULE, CassandraJamesServerMain.PROTOCOLS, CassandraJamesServerMain.PLUGINS)
+ .combineWith(ALL_BUT_JMX_CASSANDRA_MODULE)
.overrideWith(binder -> binder.bind(TextExtractor.class).to(PDFTextExtractor.class))
.overrideWith(new TestJMAPServerModule(LIMIT_TO_20_MESSAGES))
.overrideWith(new TestESMetricReporterModule())
http://git-wip-us.apache.org/repos/asf/james-project/blob/905e9425/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/cucumber/CassandraStepdefs.java
----------------------------------------------------------------------
diff --git a/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/cucumber/CassandraStepdefs.java b/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/cucumber/CassandraStepdefs.java
index f80ebe9..0b17454 100644
--- a/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/cucumber/CassandraStepdefs.java
+++ b/server/protocols/jmap-integration-testing/cassandra-jmap-integration-testing/src/test/java/org/apache/james/jmap/cassandra/cucumber/CassandraStepdefs.java
@@ -19,13 +19,14 @@
package org.apache.james.jmap.cassandra.cucumber;
+import static org.apache.james.CassandraJamesServerMain.ALL_BUT_JMX_CASSANDRA_MODULE;
+
import java.util.Arrays;
import javax.inject.Inject;
import org.apache.activemq.store.PersistenceAdapter;
import org.apache.activemq.store.memory.MemoryPersistenceAdapter;
-import org.apache.james.CassandraJamesServerMain;
import org.apache.james.GuiceJamesServer;
import org.apache.james.backends.cassandra.DockerCassandraRule;
import org.apache.james.backends.es.EmbeddedElasticSearch;
@@ -69,7 +70,7 @@ public class CassandraStepdefs {
.build();
mainStepdefs.jmapServer = new GuiceJamesServer(configuration)
- .combineWith(CassandraJamesServerMain.CASSANDRA_SERVER_MODULE, CassandraJamesServerMain.PROTOCOLS, CassandraJamesServerMain.PLUGINS)
+ .combineWith(ALL_BUT_JMX_CASSANDRA_MODULE)
.overrideWith(new CassandraJmapServerModule(embeddedElasticSearch, cassandraServer.getIp(), cassandraServer.getBindingPort()))
.overrideWith((binder) -> binder.bind(PersistenceAdapter.class).to(MemoryPersistenceAdapter.class));
mainStepdefs.awaitMethod = () -> embeddedElasticSearch.awaitForElasticSearch();
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[11/14] james-project git commit: MAILBOX-338 Implement a cache for
TikaTextExtractor
Posted by bt...@apache.org.
MAILBOX-338 Implement a cache for TikaTextExtractor
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/6d3396ef
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/6d3396ef
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/6d3396ef
Branch: refs/heads/master
Commit: 6d3396efb096665c1fae01b13b292d49b3840888
Parents: 918c86f
Author: benwa <bt...@linagora.com>
Authored: Sun May 27 13:26:43 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../mailbox/tika/CachingTextExtractor.java | 84 +++++++++
.../james/mailbox/tika/TikaConfiguration.java | 66 ++++++-
.../mailbox/tika/CachingTextExtractorTest.java | 98 ++++++++++
.../mailbox/TikaConfigurationReader.java | 64 +++++++
.../modules/mailbox/TikaMailboxModule.java | 30 ++--
.../mailbox/TikaConfigurationReaderTest.java | 178 +++++++++++++++++++
6 files changed, 504 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/6d3396ef/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
new file mode 100644
index 0000000..8bfed1b
--- /dev/null
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -0,0 +1,84 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.tika;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.james.mailbox.extractor.ParsedContent;
+import org.apache.james.mailbox.extractor.TextExtractor;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
+
+public class CachingTextExtractor implements TextExtractor {
+ private final TextExtractor underlying;
+ private final Cache<String, ParsedContent> cache;
+
+ public CachingTextExtractor(TextExtractor underlying, Duration cacheEvictionPeriod, Long cacheWeightInBytes) {
+ this.underlying = underlying;
+
+ Weigher<String, ParsedContent> weigher = (key, parsedContent) -> getSize(parsedContent);
+ this.cache = CacheBuilder.<String, String>newBuilder()
+ .expireAfterAccess(cacheEvictionPeriod.toMillis(), TimeUnit.MILLISECONDS)
+ .maximumWeight(cacheWeightInBytes)
+ .weigher(weigher)
+ .build();
+ }
+
+ private int getSize(ParsedContent parsedContent) {
+ return parsedContent.getTextualContent()
+ .map(String::length)
+ .map(this::utf16LengthToBytesCount)
+ .orElse(0);
+ }
+
+ private int utf16LengthToBytesCount(Integer value) {
+ return value * 2;
+ }
+
+ @Override
+ public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
+ byte[] bytes = IOUtils.toByteArray(inputStream);
+ String key = DigestUtils.sha256Hex(bytes);
+ try {
+ return cache.get(key,
+ () -> underlying.extractContent(new ByteArrayInputStream(bytes), contentType));
+ } catch (UncheckedExecutionException | ExecutionException e) {
+ throw unwrap(e);
+ }
+ }
+
+ private Exception unwrap(Exception e) {
+ return Optional.ofNullable(e.getCause())
+ .filter(throwable -> throwable instanceof Exception)
+ .map(throwable -> (Exception) throwable)
+ .orElse(e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/6d3396ef/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
index 633afc3..e7994b7 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/TikaConfiguration.java
@@ -19,6 +19,8 @@
package org.apache.james.mailbox.tika;
+import java.time.Duration;
+import java.util.Objects;
import java.util.Optional;
import org.apache.james.util.Port;
@@ -36,11 +38,15 @@ public class TikaConfiguration {
private Optional<String> host;
private Optional<Integer> port;
private Optional<Integer> timeoutInMillis;
+ private Optional<Duration> cacheEvictionPeriod;
+ private Optional<Long> cacheWeightInBytes;
private Builder() {
host = Optional.empty();
port = Optional.empty();
timeoutInMillis = Optional.empty();
+ cacheEvictionPeriod = Optional.empty();
+ cacheWeightInBytes = Optional.empty();
}
public Builder host(String host) {
@@ -58,24 +64,52 @@ public class TikaConfiguration {
return this;
}
+ public Builder cacheEvictionPeriod(Duration duration) {
+ this.cacheEvictionPeriod = Optional.of(duration);
+ return this;
+ }
+
+ public Builder cacheEvictionPeriod(Optional<Duration> duration) {
+ this.cacheEvictionPeriod = duration;
+ return this;
+ }
+
+ public Builder cacheWeightInBytes(long weight) {
+ this.cacheWeightInBytes = Optional.of(weight);
+ return this;
+ }
+
+ public Builder cacheWeightInBytes(Optional<Long> weight) {
+ this.cacheWeightInBytes = weight;
+ return this;
+ }
+
public TikaConfiguration build() {
Preconditions.checkState(host.isPresent(), "'host' is mandatory");
Preconditions.checkState(port.isPresent(), "'port' is mandatory");
Preconditions.checkState(timeoutInMillis.isPresent(), "'timeoutInMillis' is mandatory");
Port.assertValid(port.get());
- return new TikaConfiguration(host.get(), port.get(), timeoutInMillis.get());
+ return new TikaConfiguration(host.get(), port.get(), timeoutInMillis.get(),
+ cacheEvictionPeriod.orElse(DEFAULT_CACHE_EVICTION_PERIOD),
+ cacheWeightInBytes.orElse(DEFAULT_CACHE_LIMIT_100_MB));
}
}
+ public static final long DEFAULT_CACHE_LIMIT_100_MB = 1024L * 1024L * 100L;
+ public static final Duration DEFAULT_CACHE_EVICTION_PERIOD = Duration.ofDays(1);
private final String host;
private final int port;
private final int timeoutInMillis;
+ private final Duration cacheEvictionPeriod;
+ private final long cacheWeightInBytes;
- private TikaConfiguration(String host, int port, int timeoutInMillis) {
+ private TikaConfiguration(String host, int port, int timeoutInMillis, Duration cacheEvictionPeriod, long cacheWeightInBytes) {
this.host = host;
this.port = port;
this.timeoutInMillis = timeoutInMillis;
+ this.cacheEvictionPeriod = cacheEvictionPeriod;
+ this.cacheWeightInBytes = cacheWeightInBytes;
}
public String getHost() {
@@ -89,4 +123,32 @@ public class TikaConfiguration {
public int getTimeoutInMillis() {
return timeoutInMillis;
}
+
+ public Duration getCacheEvictionPeriod() {
+ return cacheEvictionPeriod;
+ }
+
+ public long getCacheWeightInBytes() {
+ return cacheWeightInBytes;
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ if (o instanceof TikaConfiguration) {
+ TikaConfiguration that = (TikaConfiguration) o;
+
+ return Objects.equals(this.port, that.port)
+ && Objects.equals(this.timeoutInMillis, that.timeoutInMillis)
+ && Objects.equals(this.cacheWeightInBytes, that.cacheWeightInBytes)
+ && Objects.equals(this.host, that.host)
+ && Objects.equals(this.cacheEvictionPeriod, that.cacheEvictionPeriod);
+ }
+ return false;
+ }
+
+ @Override
+ public final int hashCode() {
+ return Objects.hash(host, port, timeoutInMillis, cacheEvictionPeriod, cacheWeightInBytes);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/6d3396ef/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
new file mode 100644
index 0000000..154a5ff
--- /dev/null
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -0,0 +1,98 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.mailbox.tika;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.function.Supplier;
+
+import org.apache.james.mailbox.extractor.ParsedContent;
+import org.apache.james.mailbox.extractor.TextExtractor;
+import org.junit.Before;
+import org.junit.Test;
+import org.testcontainers.shaded.com.google.common.collect.ImmutableMap;
+
+public class CachingTextExtractorTest {
+
+ public static final ParsedContent RESULT = new ParsedContent("content", ImmutableMap.of());
+ public static final Supplier<InputStream> INPUT_STREAM_1 = () -> new ByteArrayInputStream("content1".getBytes(StandardCharsets.UTF_8));
+
+ private TextExtractor textExtractor;
+ private TextExtractor wrappedTextExtractor;
+
+ @Before
+ public void setUp() throws Exception {
+ wrappedTextExtractor = mock(TextExtractor.class);
+ textExtractor = new CachingTextExtractor(wrappedTextExtractor,
+ TikaConfiguration.DEFAULT_CACHE_EVICTION_PERIOD,
+ TikaConfiguration.DEFAULT_CACHE_LIMIT_100_MB);
+
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenReturn(RESULT);
+ }
+
+ @Test
+ public void extractContentShouldCallUnderlyingTextExtractor() throws Exception {
+ textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes");
+
+ verify(wrappedTextExtractor, times(1)).extractContent(any(), any());
+ verifyNoMoreInteractions(wrappedTextExtractor);
+ }
+
+ @Test
+ public void extractContentShouldAvoidCallingUnderlyingTextExtractorWhenPossible() throws Exception {
+ textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes");
+ textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes");
+
+ verify(wrappedTextExtractor, times(1)).extractContent(any(), any());
+ verifyNoMoreInteractions(wrappedTextExtractor);
+ }
+
+ @Test
+ public void extractContentShouldPropagateCheckedException() throws Exception {
+ IOException ioException = new IOException("Any");
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenThrow(ioException);
+
+ assertThatThrownBy(() -> textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes"))
+ .isEqualTo(ioException);
+ }
+
+ @Test
+ public void extractContentShouldPropagateRuntimeException() throws Exception {
+ RuntimeException runtimeException = new RuntimeException("Any");
+ when(wrappedTextExtractor.extractContent(any(), any()))
+ .thenThrow(runtimeException);
+
+ assertThatThrownBy(() -> textExtractor.extractContent(INPUT_STREAM_1.get(), "application/bytes"))
+ .isEqualTo(runtimeException);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/james-project/blob/6d3396ef/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
new file mode 100644
index 0000000..13ca0cd
--- /dev/null
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaConfigurationReader.java
@@ -0,0 +1,64 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules.mailbox;
+
+import java.time.Duration;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.james.mailbox.tika.TikaConfiguration;
+import org.apache.james.util.Size;
+import org.apache.james.util.TimeConverter;
+
+import com.github.fge.lambdas.Throwing;
+import com.google.common.primitives.Ints;
+
+public class TikaConfigurationReader {
+ public static final String TIKA_HOST = "tika.host";
+ public static final String TIKA_PORT = "tika.port";
+ public static final String TIKA_TIMEOUT_IN_MS = "tika.timeoutInMillis";
+ public static final String DEFAULT_HOST = "127.0.0.1";
+ public static final int DEFAULT_PORT = 9998;
+ public static final String TIKA_CACHE_EVICTION_PERIOD = "tika.cache.eviction.period";
+ public static final String TIKA_CACHE_WEIGHT_MAX = "tika.cache.weight.max";
+ public static final int DEFAULT_TIMEOUT_IN_MS = Ints.checkedCast(TimeUnit.SECONDS.toMillis(30));
+
+ public static TikaConfiguration readTikaConfiguration(PropertiesConfiguration configuration) {
+ Optional<Duration> cacheEvictionPeriod = Optional.ofNullable(
+ configuration.getString(TIKA_CACHE_EVICTION_PERIOD,
+ null))
+ .map(rawString -> TimeConverter.getMilliSeconds(rawString, TimeConverter.Unit.SECONDS))
+ .map(Duration::ofMillis);
+
+ Optional<Long> cacheWeight = Optional.ofNullable(
+ configuration.getString(TIKA_CACHE_WEIGHT_MAX, null))
+ .map(Throwing.function(Size::parse))
+ .map(Size::asBytes);
+
+ return TikaConfiguration.builder()
+ .host(configuration.getString(TIKA_HOST, DEFAULT_HOST))
+ .port(configuration.getInt(TIKA_PORT, DEFAULT_PORT))
+ .timeoutInMillis(configuration.getInt(TIKA_TIMEOUT_IN_MS, DEFAULT_TIMEOUT_IN_MS))
+ .cacheEvictionPeriod(cacheEvictionPeriod)
+ .cacheWeightInBytes(cacheWeight)
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/6d3396ef/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
index c9cc6b8..6fcd505 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
@@ -19,15 +19,19 @@
package org.apache.james.modules.mailbox;
+import static org.apache.james.modules.mailbox.TikaConfigurationReader.DEFAULT_HOST;
+import static org.apache.james.modules.mailbox.TikaConfigurationReader.DEFAULT_PORT;
+import static org.apache.james.modules.mailbox.TikaConfigurationReader.DEFAULT_TIMEOUT_IN_MS;
+
import java.io.FileNotFoundException;
import java.net.URISyntaxException;
-import java.util.concurrent.TimeUnit;
import javax.inject.Singleton;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.mailbox.tika.CachingTextExtractor;
import org.apache.james.mailbox.tika.TikaConfiguration;
import org.apache.james.mailbox.tika.TikaHttpClient;
import org.apache.james.mailbox.tika.TikaHttpClientImpl;
@@ -36,7 +40,6 @@ import org.apache.james.utils.PropertiesProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.primitives.Ints;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.Scopes;
@@ -46,18 +49,11 @@ public class TikaMailboxModule extends AbstractModule {
private static final Logger LOGGER = LoggerFactory.getLogger(TikaMailboxModule.class);
private static final String TIKA_CONFIGURATION_NAME = "tika";
- private static final String TIKA_HOST = "tika.host";
- private static final String TIKA_PORT = "tika.port";
- private static final String TIKA_TIMEOUT_IN_MS = "tika.timeoutInMillis";
- private static final String DEFAULT_HOST = "127.0.0.1";
- private static final int DEFAULT_PORT = 9998;
- private static final int DEFAULT_TIMEOUT_IN_MS = Ints.checkedCast(TimeUnit.SECONDS.toMillis(30));
@Override
protected void configure() {
bind(TikaTextExtractor.class).in(Scopes.SINGLETON);
- bind(TextExtractor.class).to(TikaTextExtractor.class);
}
@Provides
@@ -71,11 +67,8 @@ public class TikaMailboxModule extends AbstractModule {
private TikaConfiguration getTikaConfiguration(PropertiesProvider propertiesProvider) throws ConfigurationException {
try {
PropertiesConfiguration configuration = propertiesProvider.getConfiguration(TIKA_CONFIGURATION_NAME);
- return TikaConfiguration.builder()
- .host(configuration.getString(TIKA_HOST, DEFAULT_HOST))
- .port(configuration.getInt(TIKA_PORT, DEFAULT_PORT))
- .timeoutInMillis(configuration.getInt(TIKA_TIMEOUT_IN_MS, DEFAULT_TIMEOUT_IN_MS))
- .build();
+
+ return TikaConfigurationReader.readTikaConfiguration(configuration);
} catch (FileNotFoundException e) {
LOGGER.warn("Could not find {} configuration file. Using {}:{} as contact point", TIKA_CONFIGURATION_NAME, DEFAULT_HOST, DEFAULT_PORT);
return TikaConfiguration.builder()
@@ -86,4 +79,13 @@ public class TikaMailboxModule extends AbstractModule {
}
}
+ @Provides
+ @Singleton
+ private TextExtractor provideTextExtractor(TikaTextExtractor textExtractor, TikaConfiguration configuration) {
+ return new CachingTextExtractor(
+ textExtractor,
+ configuration.getCacheEvictionPeriod(),
+ configuration.getCacheWeightInBytes());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/6d3396ef/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
new file mode 100644
index 0000000..484c8f0
--- /dev/null
+++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/modules/mailbox/TikaConfigurationReaderTest.java
@@ -0,0 +1,178 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules.mailbox;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.StringReader;
+import java.time.Duration;
+
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.james.mailbox.tika.TikaConfiguration;
+import org.junit.Test;
+
+public class TikaConfigurationReaderTest {
+
+ @Test
+ public void readTikaConfigurationShouldAcceptMandatoryValues() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(500)
+ .cacheWeightInBytes(100L * 1024L *1024L)
+ .cacheEvictionPeriod(Duration.ofDays(1))
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldReturnDefaultOnMissingHost() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("127.0.0.1")
+ .port(889)
+ .timeoutInMillis(500)
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldReturnDefaultOnMissingPort() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.timeoutInMillis=500\n"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(9998)
+ .timeoutInMillis(500)
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldReturnDefaultOnMissingTimeout() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(30 * 1000)
+ .build());
+ }
+
+
+ @Test
+ public void readTikaConfigurationShouldParseUnitForCacheEvictionPeriod() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.eviction.period=2H"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(500)
+ .cacheEvictionPeriod(Duration.ofHours(2))
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldDefaultToSecondWhenMissingUnitForCacheEvitionPeriod() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.eviction.period=3600"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(500)
+ .cacheEvictionPeriod(Duration.ofHours(1))
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldParseUnitForCacheWeightMax() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.weight.max=200M"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(500)
+ .cacheWeightInBytes(200L * 1024L * 1024L)
+ .build());
+ }
+
+ @Test
+ public void readTikaConfigurationShouldDefaultToByteAsSizeUnit() throws Exception {
+ PropertiesConfiguration configuration = new PropertiesConfiguration();
+ configuration.load(new StringReader(
+ "tika.host=172.0.0.5\n" +
+ "tika.port=889\n" +
+ "tika.timeoutInMillis=500\n" +
+ "tika.cache.weight.max=1520000"));
+
+ assertThat(TikaConfigurationReader.readTikaConfiguration(configuration))
+ .isEqualTo(
+ TikaConfiguration.builder()
+ .host("172.0.0.5")
+ .port(889)
+ .timeoutInMillis(500)
+ .cacheWeightInBytes(1520000)
+ .build());
+ }
+
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[13/14] james-project git commit: MAILBOX-338 Responsibility for
weight metric update should belong to the loader
Posted by bt...@apache.org.
MAILBOX-338 Responsibility for weight metric update should belong to the loader
Not to the weigher
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/2d94eb5f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/2d94eb5f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/2d94eb5f
Branch: refs/heads/master
Commit: 2d94eb5f602082ed63e3f2866147df44c0c8aab4
Parents: f63e5d8
Author: benwa <bt...@linagora.com>
Authored: Wed May 30 09:14:20 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../mailbox/tika/CachingTextExtractor.java | 22 +++++++++++---------
1 file changed, 12 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/2d94eb5f/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
index b0bae1f..c06128f 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -39,6 +39,7 @@ import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.Weigher;
+import com.google.common.util.concurrent.UncheckedExecutionException;
public class CachingTextExtractor implements TextExtractor {
private final TextExtractor underlying;
@@ -51,14 +52,10 @@ public class CachingTextExtractor implements TextExtractor {
this.weightMetric = metricFactory.generate("textExtractor.cache.weight");
Weigher<String, ParsedContent> weigher =
- (key, parsedContent) -> {
- int size = getSize(parsedContent);
- weightMetric.add(size);
- return size;
- };
+ (key, parsedContent) -> computeWeight(parsedContent);
RemovalListener<String, ParsedContent> removalListener =
notification -> Optional.ofNullable(notification.getValue())
- .map(this::getSize)
+ .map(this::computeWeight)
.ifPresent(weightMetric::remove);
this.cache = CacheBuilder.<String, String>newBuilder()
@@ -99,7 +96,7 @@ public class CachingTextExtractor implements TextExtractor {
cache::size);
}
- private int getSize(ParsedContent parsedContent) {
+ private int computeWeight(ParsedContent parsedContent) {
return parsedContent.getTextualContent()
.map(String::length)
.map(this::utf16LengthToBytesCount)
@@ -116,13 +113,18 @@ public class CachingTextExtractor implements TextExtractor {
String key = DigestUtils.sha256Hex(bytes);
try {
- return cache.get(key,
- () -> underlying.extractContent(new ByteArrayInputStream(bytes), contentType));
- } catch (ExecutionException e) {
+ return cache.get(key, () -> retrieveAndUpdateWeight(bytes, contentType));
+ } catch (ExecutionException | UncheckedExecutionException e) {
throw unwrap(e);
}
}
+ private ParsedContent retrieveAndUpdateWeight(byte[] bytes, String contentType) throws Exception {
+ ParsedContent parsedContent = underlying.extractContent(new ByteArrayInputStream(bytes), contentType);
+ weightMetric.add(computeWeight(parsedContent));
+ return parsedContent;
+ }
+
private Exception unwrap(Exception e) {
return Optional.ofNullable(e.getCause())
.filter(throwable -> throwable instanceof Exception)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[14/14] james-project git commit: MAILBOX-338 Improve cache
management in CachingTextExtractor
Posted by bt...@apache.org.
MAILBOX-338 Improve cache management in CachingTextExtractor
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/6b00971f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/6b00971f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/6b00971f
Branch: refs/heads/master
Commit: 6b00971f3007bcd53483c9c0a29ada7c4e3c983b
Parents: 69836e4
Author: benwa <bt...@linagora.com>
Authored: Tue May 29 16:18:44 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
.../mailbox/tika/CachingTextExtractor.java | 77 ++++++++++----------
.../mailbox/tika/CachingTextExtractorTest.java | 4 +-
.../metrics/logger/DefaultMetricFactory.java | 5 --
.../modules/mailbox/TikaMailboxModule.java | 8 +-
4 files changed, 46 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/6b00971f/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
index 4429664..e2e122b 100644
--- a/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
+++ b/mailbox/tika/src/main/java/org/apache/james/mailbox/tika/CachingTextExtractor.java
@@ -23,13 +23,13 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.time.Duration;
import java.util.Optional;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.metrics.api.GaugeRegistry;
import org.apache.james.metrics.api.Metric;
import org.apache.james.metrics.api.MetricFactory;
@@ -38,14 +38,14 @@ import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.Weigher;
-import com.google.common.util.concurrent.UncheckedExecutionException;
public class CachingTextExtractor implements TextExtractor {
private final TextExtractor underlying;
private final Cache<String, ParsedContent> cache;
private final Metric weightMetric;
- public CachingTextExtractor(TextExtractor underlying, Duration cacheEvictionPeriod, Long cacheWeightInBytes, MetricFactory metricFactory) {
+ public CachingTextExtractor(TextExtractor underlying, Duration cacheEvictionPeriod, Long cacheWeightInBytes,
+ MetricFactory metricFactory, GaugeRegistry gaugeRegistry) {
this.underlying = underlying;
this.weightMetric = metricFactory.generate("textExtractor.cache.weight");
@@ -67,34 +67,35 @@ public class CachingTextExtractor implements TextExtractor {
.recordStats()
.removalListener(removalListener)
.build();
- recordStats(metricFactory);
+ recordStats(gaugeRegistry);
}
- public void recordStats(MetricFactory metricFactory) {
- metricFactory.register(
- "textExtractor.cache.hit.rate",
- () -> cache.stats().hitRate());
- metricFactory.register(
- "textExtractor.cache.hit.count",
- () -> cache.stats().hitCount());
- metricFactory.register(
- "textExtractor.cache.load.count",
- () -> cache.stats().loadCount());
- metricFactory.register(
- "textExtractor.cache.eviction.count",
- () -> cache.stats().evictionCount());
- metricFactory.register(
- "textExtractor.cache.load.exception.rate",
- () -> cache.stats().loadExceptionRate());
- metricFactory.register(
- "textExtractor.cache.load.miss.rate",
- () -> cache.stats().missRate());
- metricFactory.register(
- "textExtractor.cache.load.miss.count",
- () -> cache.stats().missCount());
- metricFactory.register(
- "textExtractor.cache.size",
- cache::size);
+ public void recordStats(GaugeRegistry gaugeRegistry) {
+ gaugeRegistry
+ .register(
+ "textExtractor.cache.hit.rate",
+ () -> cache.stats().hitRate())
+ .register(
+ "textExtractor.cache.hit.count",
+ () -> cache.stats().hitCount());
+ gaugeRegistry.register(
+ "textExtractor.cache.load.count",
+ () -> cache.stats().loadCount())
+ .register(
+ "textExtractor.cache.eviction.count",
+ () -> cache.stats().evictionCount())
+ .register(
+ "textExtractor.cache.load.exception.rate",
+ () -> cache.stats().loadExceptionRate())
+ .register(
+ "textExtractor.cache.load.miss.rate",
+ () -> cache.stats().missRate())
+ .register(
+ "textExtractor.cache.load.miss.count",
+ () -> cache.stats().missCount())
+ .register(
+ "textExtractor.cache.size",
+ cache::size);
}
private int getSize(ParsedContent parsedContent) {
@@ -112,19 +113,15 @@ public class CachingTextExtractor implements TextExtractor {
public ParsedContent extractContent(InputStream inputStream, String contentType) throws Exception {
byte[] bytes = IOUtils.toByteArray(inputStream);
String key = DigestUtils.sha256Hex(bytes);
- try {
- return cache.get(key,
- () -> underlying.extractContent(new ByteArrayInputStream(bytes), contentType));
- } catch (UncheckedExecutionException | ExecutionException e) {
- throw unwrap(e);
+
+ ParsedContent cachedValue = cache.getIfPresent(key);
+ if (cachedValue != null) {
+ return cachedValue;
}
- }
- private Exception unwrap(Exception e) {
- return Optional.ofNullable(e.getCause())
- .filter(throwable -> throwable instanceof Exception)
- .map(throwable -> (Exception) throwable)
- .orElse(e);
+ ParsedContent realValue = underlying.extractContent(new ByteArrayInputStream(bytes), contentType);
+ cache.put(key, realValue);
+ return realValue;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/james-project/blob/6b00971f/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
----------------------------------------------------------------------
diff --git a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
index 519be56..5c0ebad 100644
--- a/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
+++ b/mailbox/tika/src/test/java/org/apache/james/mailbox/tika/CachingTextExtractorTest.java
@@ -40,6 +40,7 @@ import java.util.stream.IntStream;
import org.apache.james.mailbox.extractor.ParsedContent;
import org.apache.james.mailbox.extractor.TextExtractor;
+import org.apache.james.metrics.api.NoopGaugeRegistry;
import org.apache.james.metrics.api.NoopMetricFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -68,7 +69,8 @@ public class CachingTextExtractorTest {
textExtractor = new CachingTextExtractor(wrappedTextExtractor,
TikaConfiguration.DEFAULT_CACHE_EVICTION_PERIOD,
CACHE_LIMIT_10_MiB,
- new NoopMetricFactory());
+ new NoopMetricFactory(),
+ new NoopGaugeRegistry());
when(wrappedTextExtractor.extractContent(any(), any()))
.thenReturn(RESULT);
http://git-wip-us.apache.org/repos/asf/james-project/blob/6b00971f/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
----------------------------------------------------------------------
diff --git a/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java b/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
index f7f5bb4..2997805 100644
--- a/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
+++ b/metrics/metrics-logger/src/main/java/org/apache/james/metrics/logger/DefaultMetricFactory.java
@@ -20,7 +20,6 @@ package org.apache.james.metrics.logger;
import java.util.function.Supplier;
-import org.apache.james.metrics.api.Gauge;
import org.apache.james.metrics.api.Metric;
import org.apache.james.metrics.api.MetricFactory;
import org.apache.james.metrics.api.TimeMetric;
@@ -51,8 +50,4 @@ public class DefaultMetricFactory implements MetricFactory {
}
}
- @Override
- public <T> void register(String name, Gauge<T> gauge) {
-
- }
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/6b00971f/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
index 08bcfa5..655f477 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/TikaMailboxModule.java
@@ -33,6 +33,7 @@ import org.apache.james.mailbox.tika.TikaConfiguration;
import org.apache.james.mailbox.tika.TikaHttpClient;
import org.apache.james.mailbox.tika.TikaHttpClientImpl;
import org.apache.james.mailbox.tika.TikaTextExtractor;
+import org.apache.james.metrics.api.GaugeRegistry;
import org.apache.james.metrics.api.MetricFactory;
import org.apache.james.utils.PropertiesProvider;
import org.slf4j.Logger;
@@ -77,13 +78,16 @@ public class TikaMailboxModule extends AbstractModule {
@Provides
@Singleton
- private TextExtractor provideTextExtractor(TikaTextExtractor textExtractor, TikaConfiguration configuration, MetricFactory metricFactory) {
+ private TextExtractor provideTextExtractor(TikaTextExtractor textExtractor, TikaConfiguration configuration,
+ MetricFactory metricFactory, GaugeRegistry gaugeRegistry) {
if (configuration.isEnabled() && configuration.isCacheEnabled()) {
LOGGER.info("Tika cache has been enabled.");
return new CachingTextExtractor(
textExtractor,
configuration.getCacheEvictionPeriod(),
- configuration.getCacheWeightInBytes(), metricFactory);
+ configuration.getCacheWeightInBytes(),
+ metricFactory,
+ gaugeRegistry);
}
if (configuration.isEnabled()) {
return textExtractor;
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[03/14] james-project git commit: MAILBOX-338 Add documentation
Posted by bt...@apache.org.
MAILBOX-338 Add documentation
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/10dea04b
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/10dea04b
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/10dea04b
Branch: refs/heads/master
Commit: 10dea04b22779e164b0fdcf624e6c9fbb1d70b14
Parents: 2d94eb5
Author: benwa <bt...@linagora.com>
Authored: Fri May 25 09:40:52 2018 +0700
Committer: benwa <bt...@linagora.com>
Committed: Thu May 31 08:53:16 2018 +0700
----------------------------------------------------------------------
src/site/xdoc/server/config-elasticsearch.xml | 23 ++++++++++++++++++++++
1 file changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/10dea04b/src/site/xdoc/server/config-elasticsearch.xml
----------------------------------------------------------------------
diff --git a/src/site/xdoc/server/config-elasticsearch.xml b/src/site/xdoc/server/config-elasticsearch.xml
index d1fb181..2e2812f 100644
--- a/src/site/xdoc/server/config-elasticsearch.xml
+++ b/src/site/xdoc/server/config-elasticsearch.xml
@@ -118,12 +118,35 @@
Here are the different properties:
<dl>
+ <dt><strong>tika.enabled</strong></dt>
+ <dd>Should Tika text extractor be used? <br/>
+ If true, the TikaTextExtractor will be used behind a cache. <br/>
+ If false, the DefaultTextExtractor will be used (naive implementation only supporting text).<br/>
+ Defaults to false.</dd>
+
<dt><strong>tika.host</strong></dt>
<dd>IP or domain name of your Tika server. The default value is 127.0.0.1</dd>
+
<dt><strong>tika.port</strong></dt>
<dd>Port of your tika server. The default value is 9998</dd>
+
<dt><strong>tika.timeoutInMillis</strong></dt>
<dd>Timeout when issuing request to the tika server. The default value is 3 seconds.</dd>
+
+ <dt><strong>tika.cache.eviction.period</strong></dt>
+ <dd>A cache is used to avoid, when possible, query Tika multiple time for the same attachments. <br/>
+ This entry determines how long after the last read an entry vanishes.<br/>
+ Please note that units are supported (ms - millisecond, s - second, m - minute, h - hour, d - day). Default unit is seconds. <br/>
+ Default value is <b>1 day</b></dd>
+
+ <dt><strong>tika.cache.enabled</strong></dt>
+ <dd>Should the cache be used? False by default</dd>
+
+ <dt><strong>tika.cache.weight.max</strong></dt>
+ <dd>Maximum weight of the cache.<br/>
+ A value of <b>0</b> disables the cache<br/>
+ Please note that units are supported (K for KB, M for MB, G for GB). Defaults is no units, so in bytes.<br/>
+ Default value is <b>100 MB</b>.</dd>
</dl>
Note: You can launch a tika server using this command line:
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org