You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by rc...@apache.org on 2020/05/08 03:40:25 UTC
[james-project] branch master updated (59a0b23 -> 91491b3)
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git.
from 59a0b23 JAMES-3148 DeletionTests should leverage the use of AppendResult
new da36e40 JAMES-3140 CassandraBlobStoreCache should not propagate failure upon inserts
new fd9e2de JAMES-3140 Enable easily printing executed Cassandra statements
new a354bbb JAMES-3173 Document inconsistencies in admin procedures
new 91491b3 JAMES-3140 Better handle CassandraModSeqRetries
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/james/backends/cassandra/Scenario.java | 8 +-
.../james/backends/cassandra/TestingSession.java | 36 +++++++++
.../backends/cassandra/TestingSessionTest.java | 23 +++---
.../cassandra/mail/CassandraModSeqProvider.java | 16 +---
.../mail/CassandraModSeqProviderTest.java | 57 ++++++++++++++
.../cassandra/cache/CassandraBlobStoreCache.java | 10 ++-
.../cache/CassandraBlobStoreCacheTest.java | 18 +++++
.../server/manage-guice-distributed-james.md | 86 +++++++++++++++++-----
8 files changed, 210 insertions(+), 44 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 01/04: JAMES-3140 CassandraBlobStoreCache should
not propagate failure upon inserts
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit da36e40406a4a86730b126a54aa9f9fc9a5313d5
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Apr 29 15:13:21 2020 +0700
JAMES-3140 CassandraBlobStoreCache should not propagate failure upon inserts
---
.../blob/cassandra/cache/CassandraBlobStoreCache.java | 10 +++++++++-
.../cassandra/cache/CassandraBlobStoreCacheTest.java | 18 ++++++++++++++++++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java
index fe6217e..32f5d62 100644
--- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java
+++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCache.java
@@ -38,6 +38,8 @@ import javax.inject.Inject;
import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor;
import org.apache.james.blob.api.BlobId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.Row;
@@ -48,6 +50,8 @@ import reactor.core.publisher.Mono;
public class CassandraBlobStoreCache implements BlobStoreCache {
+ public static final Logger LOGGER = LoggerFactory.getLogger(CassandraBlobStoreCache.class);
+
private final CassandraAsyncExecutor cassandraAsyncExecutor;
private final PreparedStatement insertStatement;
private final PreparedStatement selectStatement;
@@ -99,7 +103,11 @@ public class CassandraBlobStoreCache implements BlobStoreCache {
.setString(ID, blobId.asString())
.setBytes(DATA, data)
.setInt(TTL_FOR_ROW, timeToLive)
- .setConsistencyLevel(ONE));
+ .setConsistencyLevel(ONE))
+ .onErrorResume(e -> {
+ LOGGER.warn("Failed saving {} in blob store cache", blobId, e);
+ return Mono.empty();
+ });
}
private ByteBuffer toByteBuffer(byte[] bytes) {
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java
index efdc705..10c1feb 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CassandraBlobStoreCacheTest.java
@@ -18,15 +18,21 @@
****************************************************************/
package org.apache.james.blob.cassandra.cache;
+import static org.apache.james.backends.cassandra.Scenario.Builder.fail;
+import static org.assertj.core.api.Assertions.assertThatCode;
+
import java.time.Duration;
import org.apache.james.backends.cassandra.CassandraCluster;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.HashBlobId;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
+import reactor.core.publisher.Mono;
+
public class CassandraBlobStoreCacheTest implements BlobStoreCacheContract {
@RegisterExtension
@@ -59,4 +65,16 @@ public class CassandraBlobStoreCacheTest implements BlobStoreCacheContract {
public BlobId.Factory blobIdFactory() {
return blobIdFactory;
}
+
+ @Test
+ void cacheShouldNotPropagateFailures(CassandraCluster cassandra) {
+ cassandra.getConf().registerScenario(fail()
+ .forever()
+ .whenQueryStartsWith("INSERT INTO blob_cache (id,data) VALUES (:id,:data) USING TTL :ttl;"));
+
+ BlobId blobId = blobIdFactory().randomId();
+
+ assertThatCode(() -> Mono.from(testee.cache(blobId, EIGHT_KILOBYTES)).block())
+ .doesNotThrowAnyException();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 02/04: JAMES-3140 Enable easily printing executed
Cassandra statements
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit fd9e2de68282884c3d2f19d19bd0c12cfd32e9d0
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Thu May 7 16:09:06 2020 +0700
JAMES-3140 Enable easily printing executed Cassandra statements
This is helping a lot writing instrumented tests for Cassandra, and allow
easy retrieval of the statements to interact with.
I chose System output for it to be visible easily by default in the test
console, without modifying logging configuration.
---
.../james/backends/cassandra/TestingSession.java | 36 ++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java
index c8a7a54..f591286 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSession.java
@@ -21,6 +21,7 @@ package org.apache.james.backends.cassandra;
import java.util.Map;
+import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.CloseFuture;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.PreparedStatement;
@@ -28,16 +29,23 @@ import com.datastax.driver.core.RegularStatement;
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.ResultSetFuture;
import com.datastax.driver.core.Session;
+import com.datastax.driver.core.SimpleStatement;
import com.datastax.driver.core.Statement;
import com.google.common.util.concurrent.ListenableFuture;
public class TestingSession implements Session {
private final Session delegate;
private volatile Scenario scenario;
+ private volatile boolean printStatements;
public TestingSession(Session delegate) {
this.delegate = delegate;
this.scenario = Scenario.NOTHING;
+ this.printStatements = false;
+ }
+
+ public void printStatements() {
+ printStatements = true;
}
public void registerScenario(Scenario scenario) {
@@ -65,46 +73,74 @@ public class TestingSession implements Session {
@Override
public ResultSet execute(String query) {
+ printStatement(query);
return delegate.execute(query);
}
@Override
public ResultSet execute(String query, Object... values) {
+ printStatement(query);
return delegate.execute(query, values);
}
@Override
public ResultSet execute(String query, Map<String, Object> values) {
+ printStatement(query);
return delegate.execute(query, values);
}
@Override
public ResultSet execute(Statement statement) {
+ printStatement(statement);
return delegate.execute(statement);
}
@Override
public ResultSetFuture executeAsync(String query) {
+ printStatement(query);
return delegate.executeAsync(query);
}
@Override
public ResultSetFuture executeAsync(String query, Object... values) {
+ printStatement(query);
return delegate.executeAsync(query, values);
}
@Override
public ResultSetFuture executeAsync(String query, Map<String, Object> values) {
+ printStatement(query);
return delegate.executeAsync(query, values);
}
@Override
public ResultSetFuture executeAsync(Statement statement) {
+ printStatement(statement);
return scenario
.getCorrespondingBehavior(statement)
.execute(delegate, statement);
}
+ private void printStatement(String query) {
+ if (printStatements) {
+ System.out.println("Executing: " + query);
+ }
+ }
+
+ private void printStatement(Statement statement) {
+ if (printStatements) {
+ if (statement instanceof BoundStatement) {
+ BoundStatement boundStatement = (BoundStatement) statement;
+ System.out.println("Executing: " + boundStatement.preparedStatement().getQueryString());
+ } else if (statement instanceof SimpleStatement) {
+ SimpleStatement simpleStatement = (SimpleStatement) statement;
+ System.out.println("Executing: " + simpleStatement.getQueryString());
+ } else {
+ System.out.println("Executing: " + statement);
+ }
+ }
+ }
+
@Override
public PreparedStatement prepare(String query) {
return delegate.prepare(query);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 04/04: JAMES-3140 Better handle
CassandraModSeqRetries
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 91491b393a5e80248d8e4f150d7868dfcf825a46
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Apr 29 15:16:21 2020 +0700
JAMES-3140 Better handle CassandraModSeqRetries
In an empty modseq, retires did not include the initial row creation, but
only its update.
(Triggered because "100 mail should be well received" runs way faster
with the blob store cache.)
---
.../apache/james/backends/cassandra/Scenario.java | 8 ++-
.../backends/cassandra/TestingSessionTest.java | 23 ++++-----
.../cassandra/mail/CassandraModSeqProvider.java | 16 ++----
.../mail/CassandraModSeqProviderTest.java | 57 ++++++++++++++++++++++
4 files changed, 79 insertions(+), 25 deletions(-)
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/Scenario.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/Scenario.java
index f2f3682..da3758d 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/Scenario.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/Scenario.java
@@ -33,10 +33,16 @@ import com.datastax.driver.core.Statement;
import com.google.common.base.Preconditions;
public class Scenario {
+ public static class InjectedFailureException extends RuntimeException {
+ public InjectedFailureException() {
+ super("Injected failure");
+ }
+ }
+
@FunctionalInterface
interface Behavior {
Behavior THROW = (session, statement) -> {
- throw new RuntimeException("Injected failure");
+ throw new InjectedFailureException();
};
Behavior EXECUTE_NORMALLY = Session::executeAsync;
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSessionTest.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSessionTest.java
index 6c8dfc5..54fca6f 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSessionTest.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/TestingSessionTest.java
@@ -30,6 +30,7 @@ import static org.assertj.core.api.Assertions.assertThatCode;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import org.apache.james.backends.cassandra.Scenario.Barrier;
+import org.apache.james.backends.cassandra.Scenario.InjectedFailureException;
import org.apache.james.backends.cassandra.utils.CassandraAsyncExecutor;
import org.apache.james.backends.cassandra.versions.CassandraSchemaVersionDAO;
import org.apache.james.backends.cassandra.versions.CassandraSchemaVersionModule;
@@ -89,7 +90,7 @@ class TestingSessionTest {
.whenQueryStartsWith("SELECT value FROM schemaVersion;"));
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
}
@Test
@@ -102,7 +103,7 @@ class TestingSessionTest {
assertThatThrownBy(() -> new CassandraAsyncExecutor(cassandra.getConf())
.execute(select(VALUE).from(TABLE_NAME))
.block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
}
@Test
@@ -113,7 +114,7 @@ class TestingSessionTest {
.forAllQueries());
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
}
@Test
@@ -142,9 +143,9 @@ class TestingSessionTest {
SoftAssertions.assertSoftly(softly -> {
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
assertThatCode(() -> dao.getCurrentSchemaVersion().block())
.doesNotThrowAnyException();
});
@@ -165,7 +166,7 @@ class TestingSessionTest {
assertThatCode(() -> dao.getCurrentSchemaVersion().block())
.doesNotThrowAnyException();
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
assertThatCode(() -> dao.getCurrentSchemaVersion().block())
.doesNotThrowAnyException();
});
@@ -180,11 +181,11 @@ class TestingSessionTest {
SoftAssertions.assertSoftly(softly -> {
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
});
}
@@ -198,7 +199,7 @@ class TestingSessionTest {
dao.updateVersion(new SchemaVersion(36)).block();
assertThatThrownBy(() -> dao.getCurrentSchemaVersion().block())
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
}
@Test
@@ -287,6 +288,6 @@ class TestingSessionTest {
barrier.releaseCaller();
assertThatThrownBy(operation::block)
- .isInstanceOf(RuntimeException.class);
+ .isInstanceOf(InjectedFailureException.class);
}
}
diff --git a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProvider.java b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProvider.java
index f3ee258..15f1a2d 100644
--- a/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProvider.java
+++ b/mailbox/cassandra/src/main/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProvider.java
@@ -56,7 +56,6 @@ import reactor.util.retry.Retry;
public class CassandraModSeqProvider implements ModSeqProvider {
public static final String MOD_SEQ_CONDITION = "modSeqCondition";
- private final long maxModSeqRetries;
public static class ExceptionRelay extends RuntimeException {
private final MailboxException underlying;
@@ -83,6 +82,7 @@ public class CassandraModSeqProvider implements ModSeqProvider {
}
private final CassandraAsyncExecutor cassandraAsyncExecutor;
+ private final long maxModSeqRetries;
private final PreparedStatement select;
private final PreparedStatement update;
private final PreparedStatement insert;
@@ -178,24 +178,14 @@ public class CassandraModSeqProvider implements ModSeqProvider {
}
public Mono<ModSeq> nextModSeq(CassandraId mailboxId) {
+ Duration firstBackoff = Duration.ofMillis(10);
+
return findHighestModSeq(mailboxId)
.flatMap(maybeHighestModSeq -> maybeHighestModSeq
.map(highestModSeq -> tryUpdateModSeq(mailboxId, highestModSeq))
.orElseGet(() -> tryInsertModSeq(mailboxId, ModSeq.first())))
- .switchIfEmpty(handleRetries(mailboxId));
- }
-
- private Mono<ModSeq> handleRetries(CassandraId mailboxId) {
- Duration firstBackoff = Duration.ofMillis(10);
- return tryFindThenUpdateOnce(mailboxId)
.single()
.retryWhen(Retry.backoff(maxModSeqRetries, firstBackoff).scheduler(Schedulers.elastic()));
}
- private Mono<ModSeq> tryFindThenUpdateOnce(CassandraId mailboxId) {
- return Mono.defer(() -> findHighestModSeq(mailboxId)
- .<ModSeq>handle((t, sink) -> t.ifPresent(sink::next))
- .flatMap(highestModSeq -> tryUpdateModSeq(mailboxId, highestModSeq)));
- }
-
}
diff --git a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProviderTest.java b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProviderTest.java
index 6c328a4..e3aed85 100644
--- a/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProviderTest.java
+++ b/mailbox/cassandra/src/test/java/org/apache/james/mailbox/cassandra/mail/CassandraModSeqProviderTest.java
@@ -18,7 +18,13 @@
****************************************************************/
package org.apache.james.mailbox.cassandra.mail;
+import static org.apache.james.backends.cassandra.Scenario.Builder.awaitOn;
+import static org.apache.james.backends.cassandra.Scenario.Builder.executeNormally;
+import static org.apache.james.mailbox.cassandra.table.CassandraMessageModseqTable.MAILBOX_ID;
+import static org.apache.james.mailbox.cassandra.table.CassandraMessageModseqTable.TABLE_NAME;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.time.Duration;
import java.util.concurrent.ConcurrentSkipListSet;
@@ -27,6 +33,8 @@ import java.util.stream.LongStream;
import org.apache.james.backends.cassandra.CassandraCluster;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
+import org.apache.james.backends.cassandra.Scenario;
+import org.apache.james.backends.cassandra.Scenario.Barrier;
import org.apache.james.backends.cassandra.init.configuration.CassandraConfiguration;
import org.apache.james.core.Username;
import org.apache.james.mailbox.ModSeq;
@@ -40,8 +48,12 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
+import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.github.fge.lambdas.Throwing;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+
class CassandraModSeqProviderTest {
private static final CassandraId CASSANDRA_ID = new CassandraId.Factory().fromString("e22b3ac0-a80b-11e7-bb00-777268d65503");
@@ -84,6 +96,51 @@ class CassandraModSeqProviderTest {
}
@Test
+ void failedInsertsShouldBeRetried(CassandraCluster cassandra) throws Exception {
+ Barrier insertBarrier = new Barrier(2);
+ Barrier retryBarrier = new Barrier(1);
+ cassandra.getConf()
+ .registerScenario(
+ executeNormally()
+ .times(2)
+ .whenQueryStartsWith("SELECT nextModseq FROM modseq WHERE mailboxId=:mailboxId;"),
+ awaitOn(insertBarrier)
+ .thenExecuteNormally()
+ .times(2)
+ .whenQueryStartsWith("INSERT INTO modseq (nextModseq,mailboxId) VALUES (:nextModseq,:mailboxId) IF NOT EXISTS;"),
+ awaitOn(retryBarrier)
+ .thenExecuteNormally()
+ .times(1)
+ .whenQueryStartsWith("SELECT nextModseq FROM modseq WHERE mailboxId=:mailboxId;"));
+
+ Mono<ModSeq> operation1 = modSeqProvider.nextModSeq(CASSANDRA_ID)
+ .subscribeOn(Schedulers.elastic())
+ .cache();
+ Mono<ModSeq> operation2 = modSeqProvider.nextModSeq(CASSANDRA_ID)
+ .subscribeOn(Schedulers.elastic())
+ .cache();
+
+ operation1.subscribe();
+ operation2.subscribe();
+
+ insertBarrier.awaitCaller();
+ insertBarrier.releaseCaller();
+
+ retryBarrier.awaitCaller();
+ retryBarrier.releaseCaller();
+
+ // Artificially fail the insert failure
+ cassandra.getConf()
+ .execute(QueryBuilder.delete().from(TABLE_NAME)
+ .where(QueryBuilder.eq(MAILBOX_ID, CASSANDRA_ID.asUuid())));
+
+ retryBarrier.releaseCaller();
+
+ assertThatCode(() -> operation1.block(Duration.ofSeconds(1))).doesNotThrowAnyException();
+ assertThatCode(() -> operation2.block(Duration.ofSeconds(1))).doesNotThrowAnyException();
+ }
+
+ @Test
void nextModSeqShouldGenerateUniqueValuesWhenParallelCalls() throws ExecutionException, InterruptedException {
int nbEntries = 10;
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 03/04: JAMES-3173 Document inconsistencies in admin
procedures
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit a354bbb41a5637088ca701c2cc611ddaf22d4859
Author: LanKhuat <kh...@gmail.com>
AuthorDate: Thu May 7 14:18:03 2020 +0700
JAMES-3173 Document inconsistencies in admin procedures
---
.../server/manage-guice-distributed-james.md | 86 +++++++++++++++++-----
1 file changed, 68 insertions(+), 18 deletions(-)
diff --git a/src/site/markdown/server/manage-guice-distributed-james.md b/src/site/markdown/server/manage-guice-distributed-james.md
index c51a18e..b6013df 100644
--- a/src/site/markdown/server/manage-guice-distributed-james.md
+++ b/src/site/markdown/server/manage-guice-distributed-james.md
@@ -278,25 +278,12 @@ Because of the lack of transactions, it's hard to prevent these kind of issues.
fix some existing cassandra inconsistency issues that had been reported to James.
Here is the list of known inconsistencies:
- - [RRT (RecipientRewriteTable) mapping sources](#Rrt_RecipientRewriteTable_mapping_sources)
- [Jmap message fast view projections](#Jmap_message_fast_view_projections)
- [Mailboxes](#Mailboxes)
-
-### RRT (RecipientRewriteTable) mapping sources
-
-`rrt` and `mappings_sources` tables store information about address mappings.
-The source of truth is `rrt` and `mappings_sources` is the projection table containing all
-mapping sources.
-
-#### How to detect the inconsistencies
-
-Right now there's no tool for detecting that, we're proposing a [development plan](https://issues.apache.org/jira/browse/JAMES-3069).
-By the mean time, the recommendation is to execute the `SolveInconsistencies` task below
-in a regular basis.
-
-#### How to solve
-
-Execute the Cassandra mapping `SolveInconsistencies` task described in [webadmin documentation](manage-webadmin.html#Operations_on_mappings_sources)
+ - [Mailboxes Counters](#Mailboxes_counters)
+ - [Messages](#Messages)
+ - [Quotas](#Quotas)
+ - [RRT (RecipientRewriteTable) mapping sources](#Rrt_RecipientRewriteTable_mapping_sources)
### Jmap message fast view projections
@@ -310,7 +297,7 @@ You can watch the `MessageFastViewProjection` health check at [webadmin document
It provides a check based on the ratio of missed projection reads.
#### How to solve
-
+
Since the MessageFastViewProjection is self healing, you should be concerned only if
the health check still returns `degraded` for a while, there's a possible thing you
can do is looking at James logs for more clues.
@@ -337,6 +324,69 @@ to sanitize his mailbox denormalization.
In order to ensure being offline, stop the traffic on SMTP, JMAP and IMAP ports, for example via re-configuration or
firewall rules.
+### Mailboxes Counters
+
+James maintains a per mailbox projection for message count and unseen message count. Failures during the denormalization
+process will lead to incorrect results being returned.
+
+#### How to detect the inconsistencies
+
+Incorrect message count/message unseen count could be seen in the `Mail User Agent` (IMAP or JMAP). Invalid values are reported in the logs
+as warning with the following class `org.apache.james.mailbox.model.MailboxCounters` and the following message prefix: `Invalid mailbox counters`.
+
+#### How to solve
+
+Execute the [recompute Mailbox counters task](manage-webadmin.html#Recomputing mailbox counters).
+This task is not concurrent-safe. Concurrent increments & decrements will be ignored during a single mailbox processing.
+Re-running this task may eventually return the correct result.
+
+### Messages
+
+Messages are denormalized and stored in both `imapUidTable` (source of truth) and `messageIdTable`. Failure in the denormalization
+process will cause inconsistencies between the two tables.
+
+#### How to detect the inconsistencies
+
+User can see a message in JMAP but not in IMAP, or mark a message as 'SEEN' in JMAP but the message flag is still unchanged in IMAP.
+
+#### How to solve
+
+Execute the [solve Cassandra message inconsistencies task](manage-webadmin.html#Fixing_messages_inconsistencies).
+This task is not concurrent-safe. User actions concurrent to the inconsistency fixing task could result in new inconsistencies
+being created. However the source of truth `imapUidTable` will not be affected and thus re-running this task may eventually
+fix all issues.
+
+### Quotas
+
+User can monitor the amount of space and message count he is allowed to use, and that he is effectively using. James relies on
+an event bus and Cassandra to track the quota of an user. Upon Cassandra failure, this value can be incorrect.
+
+#### How to detect the inconsistencies
+
+Incorrect quotas could be seen in the `Mail User Agent` (IMAP or JMAP).
+
+#### How to solve
+
+Execute the [recompute Quotas counters task](manage-webadmin.html#Recomputing current quotas for users).
+This task is not concurrent-safe. Concurrent operations will result in an invalid quota to be persisted. Re-running this task may
+eventually return the correct result.
+
+### RRT (RecipientRewriteTable) mapping sources
+
+`rrt` and `mappings_sources` tables store information about address mappings.
+The source of truth is `rrt` and `mappings_sources` is the projection table containing all
+mapping sources.
+
+#### How to detect the inconsistencies
+
+Right now there's no tool for detecting that, we're proposing a [development plan](https://issues.apache.org/jira/browse/JAMES-3069).
+By the mean time, the recommendation is to execute the `SolveInconsistencies` task below
+in a regular basis.
+
+#### How to solve
+
+Execute the Cassandra mapping `SolveInconsistencies` task described in [webadmin documentation](manage-webadmin.html#Operations_on_mappings_sources)
+
## Setting Cassandra user permissions
When a Cassandra cluster is serving more than a James cluster, the keyspaces need isolation.
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org