You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/01/05 09:17:29 UTC
[03/15] james-project git commit: JAMES-2266 Integration test when
fixing ghost mailbox bug
JAMES-2266 Integration test when fixing ghost mailbox bug
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/8da3ad0b
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/8da3ad0b
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/8da3ad0b
Branch: refs/heads/master
Commit: 8da3ad0b20773b30b38483b7e71892dece37907b
Parents: fde3336
Author: benwa <bt...@linagora.com>
Authored: Wed Dec 27 17:03:18 2017 +0700
Committer: benwa <bt...@linagora.com>
Committed: Fri Jan 5 16:06:36 2018 +0700
----------------------------------------------------------------------
.../modules/mailbox/CassandraSessionModule.java | 4 +
.../mailbox/ResilientClusterProvider.java | 3 +-
.../org/apache/james/server/CassandraProbe.java | 40 +++
.../apache/james/FixingGhostMailboxTest.java | 337 +++++++++++++++++++
4 files changed, 383 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/8da3ad0b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java
index 84a33db..523cbc8 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/CassandraSessionModule.java
@@ -39,7 +39,9 @@ import org.apache.james.backends.cassandra.versions.CassandraSchemaVersionManage
import org.apache.james.backends.cassandra.versions.CassandraSchemaVersionModule;
import org.apache.james.lifecycle.api.Configurable;
import org.apache.james.mailbox.store.BatchSizes;
+import org.apache.james.server.CassandraProbe;
import org.apache.james.utils.ConfigurationPerformer;
+import org.apache.james.utils.GuiceProbe;
import org.apache.james.utils.PropertiesProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,6 +93,8 @@ public class CassandraSessionModule extends AbstractModule {
bind(CassandraSchemaVersionDAO.class).in(Scopes.SINGLETON);
Multibinder.newSetBinder(binder(), ConfigurationPerformer.class).addBinding().to(CassandraSchemaChecker.class);
+
+ Multibinder.newSetBinder(binder(), GuiceProbe.class).addBinding().to(CassandraProbe.class);
}
@Provides
http://git-wip-us.apache.org/repos/asf/james-project/blob/8da3ad0b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ResilientClusterProvider.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ResilientClusterProvider.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ResilientClusterProvider.java
index 93adf63..56820ec 100644
--- a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ResilientClusterProvider.java
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/modules/mailbox/ResilientClusterProvider.java
@@ -55,6 +55,7 @@ import com.nurkiewicz.asyncretry.function.RetryCallable;
@Singleton
public class ResilientClusterProvider implements Provider<Cluster> {
+ public static final String CASSANDRA_KEYSPACE = "cassandra.keyspace";
private static final int DEFAULT_CONNECTION_MAX_RETRIES = 10;
private static final int DEFAULT_CONNECTION_MIN_DELAY = 5000;
private static final long CASSANDRA_HIGHEST_TRACKABLE_LATENCY_MILLIS = TimeUnit.SECONDS.toMillis(10);
@@ -96,7 +97,7 @@ public class ResilientClusterProvider implements Provider<Cluster> {
try {
return ClusterWithKeyspaceCreatedFactory
.config(cluster,
- configuration.getString("cassandra.keyspace", DEFAULT_KEYSPACE))
+ configuration.getString(CASSANDRA_KEYSPACE, DEFAULT_KEYSPACE))
.replicationFactor(configuration.getInt("cassandra.replication.factor", DEFAULT_REPLICATION_FACTOR))
.clusterWithInitializedKeyspace();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/james-project/blob/8da3ad0b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/server/CassandraProbe.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/main/java/org/apache/james/server/CassandraProbe.java b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/server/CassandraProbe.java
new file mode 100644
index 0000000..ff03013
--- /dev/null
+++ b/server/container/guice/cassandra-guice/src/main/java/org/apache/james/server/CassandraProbe.java
@@ -0,0 +1,40 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.server;
+
+import javax.inject.Inject;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.james.backends.cassandra.init.CassandraSessionConfiguration;
+import org.apache.james.modules.mailbox.ResilientClusterProvider;
+import org.apache.james.utils.GuiceProbe;
+
+public class CassandraProbe implements GuiceProbe {
+ private final CassandraSessionConfiguration configuration;
+
+ @Inject
+ public CassandraProbe(CassandraSessionConfiguration configuration) {
+ this.configuration = configuration;
+ }
+
+ public String getKeyspace() throws ConfigurationException {
+ return configuration.getConfiguration().getString(ResilientClusterProvider.CASSANDRA_KEYSPACE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/james-project/blob/8da3ad0b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/FixingGhostMailboxTest.java
----------------------------------------------------------------------
diff --git a/server/container/guice/cassandra-guice/src/test/java/org/apache/james/FixingGhostMailboxTest.java b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/FixingGhostMailboxTest.java
new file mode 100644
index 0000000..62a4b30
--- /dev/null
+++ b/server/container/guice/cassandra-guice/src/test/java/org/apache/james/FixingGhostMailboxTest.java
@@ -0,0 +1,337 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james;
+
+import static com.datastax.driver.core.querybuilder.QueryBuilder.delete;
+import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
+import static com.jayway.restassured.RestAssured.given;
+import static com.jayway.restassured.RestAssured.with;
+import static com.jayway.restassured.config.EncoderConfig.encoderConfig;
+import static com.jayway.restassured.config.RestAssuredConfig.newConfig;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Date;
+
+import javax.mail.Flags;
+
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.james.backends.cassandra.ContainerLifecycleConfiguration;
+import org.apache.james.backends.cassandra.init.CassandraTypesProvider;
+import org.apache.james.jmap.HttpJmapAuthentication;
+import org.apache.james.jmap.api.access.AccessToken;
+import org.apache.james.mailbox.cassandra.mail.task.MailboxMergingTask;
+import org.apache.james.mailbox.cassandra.mail.utils.MailboxBaseTupleUtil;
+import org.apache.james.mailbox.cassandra.modules.CassandraMailboxModule;
+import org.apache.james.mailbox.cassandra.table.CassandraMailboxPathTable;
+import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.model.ComposedMessageId;
+import org.apache.james.mailbox.model.MailboxACL;
+import org.apache.james.mailbox.model.MailboxConstants;
+import org.apache.james.mailbox.model.MailboxId;
+import org.apache.james.mailbox.model.MailboxPath;
+import org.apache.james.mailbox.store.mail.model.Mailbox;
+import org.apache.james.mailbox.store.probe.ACLProbe;
+import org.apache.james.mailbox.store.probe.MailboxProbe;
+import org.apache.james.modules.ACLProbeImpl;
+import org.apache.james.modules.MailboxProbeImpl;
+import org.apache.james.probe.DataProbe;
+import org.apache.james.server.CassandraProbe;
+import org.apache.james.task.TaskManager;
+import org.apache.james.utils.DataProbeImpl;
+import org.apache.james.utils.JmapGuiceProbe;
+import org.apache.james.utils.WebAdminGuiceProbe;
+import org.apache.james.webadmin.RandomPortSupplier;
+import org.apache.james.webadmin.WebAdminConfiguration;
+import org.apache.james.webadmin.routes.CassandraMailboxMergingRoutes;
+import org.apache.james.webadmin.routes.TasksRoutes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.Session;
+import com.google.common.base.Charsets;
+import com.jayway.restassured.RestAssured;
+import com.jayway.restassured.builder.RequestSpecBuilder;
+import com.jayway.restassured.http.ContentType;
+
+public class FixingGhostMailboxTest {
+
+ private static final String NAME = "[0][0]";
+ private static final String ARGUMENTS = "[0][1]";
+ private static final String FIRST_MAILBOX = ARGUMENTS + ".list[0]";
+ public static final boolean RECENT = true;
+
+ @ClassRule
+ public static DockerCassandraRule cassandra = new DockerCassandraRule();
+
+ public static ContainerLifecycleConfiguration cassandraLifecycleConfiguration = ContainerLifecycleConfiguration.withDefaultIterationsBetweenRestart().container(cassandra.getRawContainer()).build();
+
+ @Rule
+ public CassandraJmapTestRule rule = CassandraJmapTestRule.defaultTestRule();
+
+ @Rule
+ public TestRule cassandraLifecycleTestRule = cassandraLifecycleConfiguration.asTestRule();
+
+ private AccessToken accessToken;
+ private String domain;
+ private String alice;
+ private String bob;
+ private String cedric;
+ private GuiceJamesServer jmapServer;
+ private MailboxProbe mailboxProbe;
+ private ACLProbe aclProbe;
+ private Session session;
+ private CassandraTypesProvider cassandraTypesProvider;
+ private MailboxBaseTupleUtil mailboxBaseTupleUtil;
+ private ComposedMessageId message1;
+ private MailboxId aliceGhostInboxId;
+ private MailboxPath aliceInboxPath;
+ private ComposedMessageId message2;
+ private WebAdminGuiceProbe webAdminProbe;
+
+ @Before
+ public void setup() throws Throwable {
+ jmapServer = rule.jmapServer(cassandra.getModule(),
+ binder -> binder.bind(WebAdminConfiguration.class)
+ .toInstance(WebAdminConfiguration.builder()
+ .port(new RandomPortSupplier())
+ .enabled()
+ .build()));
+ jmapServer.start();
+ webAdminProbe = jmapServer.getProbe(WebAdminGuiceProbe.class);
+ mailboxProbe = jmapServer.getProbe(MailboxProbeImpl.class);
+ aclProbe = jmapServer.getProbe(ACLProbeImpl.class);
+
+ RestAssured.requestSpecification = new RequestSpecBuilder()
+ .setContentType(ContentType.JSON)
+ .setAccept(ContentType.JSON)
+ .setConfig(newConfig().encoderConfig(encoderConfig().defaultContentCharset(Charsets.UTF_8)))
+ .setPort(jmapServer.getProbe(JmapGuiceProbe.class).getJmapPort())
+ .build();
+ RestAssured.enableLoggingOfRequestAndResponseIfValidationFails();
+
+ domain = "domain.tld";
+ alice = "alice@" + domain;
+ String alicePassword = "aliceSecret";
+ bob = "bob@" + domain;
+ cedric = "cedric@" + domain;
+ DataProbe dataProbe = jmapServer.getProbe(DataProbeImpl.class);
+ dataProbe.addDomain(domain);
+ dataProbe.addUser(alice, alicePassword);
+ dataProbe.addUser(bob, "bobSecret");
+ accessToken = HttpJmapAuthentication.authenticateJamesUser(baseUri(), alice, alicePassword);
+
+ session = Cluster.builder()
+ .addContactPoint(cassandra.getIp())
+ .withPort(cassandra.getMappedPort(9042))
+ .build()
+ .connect(jmapServer.getProbe(CassandraProbe.class).getKeyspace());
+ cassandraTypesProvider = new CassandraTypesProvider(new CassandraMailboxModule(), session);
+ mailboxBaseTupleUtil = new MailboxBaseTupleUtil(cassandraTypesProvider);
+
+ simulateGhostMailboxBug();
+ }
+
+ private void simulateGhostMailboxBug() throws MailboxException {
+ // State before ghost mailbox bug
+ // Alice INBOX is delegated to Bob and contains one message
+ aliceInboxPath = MailboxPath.forUser(alice, MailboxConstants.INBOX);
+ aliceGhostInboxId = mailboxProbe.createMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+ aclProbe.addRights(aliceInboxPath, bob, MailboxACL.FULL_RIGHTS);
+ message1 = mailboxProbe.appendMessage(alice, aliceInboxPath,
+ generateMessageContent(), new Date(), !RECENT, new Flags());
+ rule.await();
+
+ // Simulate ghost mailbox bug
+ session.execute(delete().from(CassandraMailboxPathTable.TABLE_NAME)
+ .where(eq(CassandraMailboxPathTable.NAMESPACE_AND_USER, mailboxBaseTupleUtil.createMailboxBaseUDT(MailboxConstants.USER_NAMESPACE, alice)))
+ .and(eq(CassandraMailboxPathTable.MAILBOX_NAME, MailboxConstants.INBOX)));
+
+ // trigger provisioning
+ given()
+ .header("Authorization", accessToken.serialize())
+ .body("[[\"getMailboxes\", {}, \"#0\"]]")
+ .when()
+ .post("/jmap")
+ .then()
+ .statusCode(200);
+
+ // Received a new message
+ message2 = mailboxProbe.appendMessage(alice, aliceInboxPath,
+ generateMessageContent(), new Date(), !RECENT, new Flags());
+ rule.await();
+ }
+
+ private ByteArrayInputStream generateMessageContent() {
+ return new ByteArrayInputStream("Subject: toto\r\n\r\ncontent".getBytes(StandardCharsets.UTF_8));
+ }
+
+ private URIBuilder baseUri() {
+ return new URIBuilder()
+ .setScheme("http")
+ .setHost("localhost")
+ .setPort(jmapServer.getProbe(JmapGuiceProbe.class)
+ .getJmapPort())
+ .setCharset(Charsets.UTF_8);
+ }
+
+ @After
+ public void teardown() {
+ jmapServer.stop();
+ }
+
+ @Test
+ public void ghostMailboxBugShouldChangeMailboxId() throws Exception {
+ Mailbox newAliceInbox = mailboxProbe.getMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+
+ assertThat(aliceGhostInboxId).isNotEqualTo(newAliceInbox.getMailboxId());
+ }
+
+ @Test
+ public void ghostMailboxBugShouldDiscardOldContent() throws Exception {
+ Mailbox newAliceInbox = mailboxProbe.getMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+
+ given()
+ .header("Authorization", accessToken.serialize())
+ .body("[[\"getMessageList\", {\"filter\":{\"inMailboxes\":[\"" + newAliceInbox.getMailboxId().serialize() + "\"]}}, \"#0\"]]")
+ .when()
+ .post("/jmap")
+ .then()
+ .statusCode(200)
+ .body(NAME, equalTo("messageList"))
+ .body(ARGUMENTS + ".messageIds", hasSize(1))
+ .body(ARGUMENTS + ".messageIds", not(contains(message1.getMessageId().serialize())))
+ .body(ARGUMENTS + ".messageIds", contains(message2.getMessageId().serialize()));
+ }
+
+ @Test
+ public void webadminCanMergeTwoMailboxes() throws Exception {
+ Mailbox newAliceInbox = mailboxProbe.getMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+
+ fixGhostMailboxes(newAliceInbox);
+
+ given()
+ .header("Authorization", accessToken.serialize())
+ .body("[[\"getMessageList\", {\"filter\":{\"inMailboxes\":[\"" + newAliceInbox.getMailboxId().serialize() + "\"]}}, \"#0\"]]")
+ .when()
+ .post("/jmap")
+ .then()
+ .statusCode(200)
+ .body(NAME, equalTo("messageList"))
+ .body(ARGUMENTS + ".messageIds", hasSize(2))
+ .body(ARGUMENTS + ".messageIds", containsInAnyOrder(
+ message1.getMessageId().serialize(),
+ message2.getMessageId().serialize()));
+ }
+
+ @Test
+ public void webadminCanMergeTwoMailboxesRights() throws Exception {
+ Mailbox newAliceInbox = mailboxProbe.getMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+ aclProbe.addRights(aliceInboxPath, cedric, MailboxACL.FULL_RIGHTS);
+
+ fixGhostMailboxes(newAliceInbox);
+
+ given()
+ .header("Authorization", accessToken.serialize())
+ .body("[[\"getMailboxes\", {\"ids\": [\"" + newAliceInbox.getMailboxId().serialize() + "\"]}, \"#0\"]]")
+ .when()
+ .post("/jmap")
+ .then()
+ .statusCode(200)
+ .body(NAME, equalTo("mailboxes"))
+ .body(FIRST_MAILBOX + ".sharedWith", hasKey(bob))
+ .body(FIRST_MAILBOX + ".sharedWith", hasKey(cedric));
+ }
+
+ @Test
+ public void oldGhostedMailboxShouldNoMoreBeAccessible() throws Exception {
+ Mailbox newAliceInbox = mailboxProbe.getMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+ aclProbe.addRights(aliceInboxPath, cedric, MailboxACL.FULL_RIGHTS);
+
+ fixGhostMailboxes(newAliceInbox);
+
+ given()
+ .header("Authorization", accessToken.serialize())
+ .body("[[\"getMailboxes\", {\"ids\": [\"" + aliceGhostInboxId.serialize() + "\"]}, \"#0\"]]")
+ .when()
+ .post("/jmap")
+ .then()
+ .statusCode(200)
+ .body(NAME, equalTo("mailboxes"))
+ .body(ARGUMENTS + ".list", hasSize(0));
+ }
+
+ @Test
+ public void mergingMailboxTaskShouldBeInformative() {
+ Mailbox newAliceInbox = mailboxProbe.getMailbox(MailboxConstants.USER_NAMESPACE, alice, MailboxConstants.INBOX);
+
+ String taskId = fixGhostMailboxes(newAliceInbox);
+
+ given()
+ .port(webAdminProbe.getWebAdminPort())
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is(TaskManager.Status.COMPLETED.getValue()))
+ .body("taskId", is(taskId))
+ .body("additionalInformation.oldMailboxId", is(aliceGhostInboxId.serialize()))
+ .body("additionalInformation.newMailboxId", is(newAliceInbox.getMailboxId().serialize()))
+ .body("type", is(MailboxMergingTask.MAILBOX_MERGING))
+ .body("submitDate", is(not(nullValue())))
+ .body("startedDate", is(not(nullValue())))
+ .body("completedDate", is(not(nullValue())));
+
+ }
+
+ private String fixGhostMailboxes(Mailbox newAliceInbox) {
+ String taskId = with()
+ .port(webAdminProbe.getWebAdminPort())
+ .basePath(CassandraMailboxMergingRoutes.BASE)
+ .body("{" +
+ " \"mergeOrigin\":\"" + aliceGhostInboxId.serialize() + "\"," +
+ " \"mergeDestination\":\"" + newAliceInbox.getMailboxId().serialize() + "\"" +
+ "}")
+ .post()
+ .jsonPath()
+ .getString("taskId");
+ with()
+ .port(webAdminProbe.getWebAdminPort())
+ .basePath(TasksRoutes.BASE)
+ .get(taskId + "/await");
+ rule.await();
+ return taskId;
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org