You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2019/05/20 09:40:01 UTC
[james-project] 04/09: JAMES-2765 Type naming is no more supported
in ES 6
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 0ad1f0d061b73c584f73f65bc84847af49109e18
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Fri May 17 11:58:02 2019 +0700
JAMES-2765 Type naming is no more supported in ES 6
---
.../james/backends/es/v6/ElasticSearchIndexer.java | 23 +++++++---------
.../james/backends/es/v6/NodeMappingFactory.java | 12 ++++----
.../org/apache/james/backends/es/v6/TypeName.java | 32 ----------------------
.../backends/es/v6/DockerElasticSearchRule.java | 4 +++
.../backends/es/v6/ElasticSearchIndexerTest.java | 10 +------
.../backends/es/v6/NodeMappingFactoryTest.java | 14 +++-------
.../backends/es/v6/search/ScrollIterableTest.java | 32 +++++++++++++---------
7 files changed, 44 insertions(+), 83 deletions(-)
diff --git a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java
index 79adb69..9689c1a 100644
--- a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java
+++ b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/ElasticSearchIndexer.java
@@ -51,23 +51,19 @@ public class ElasticSearchIndexer {
private final RestHighLevelClient client;
private final AliasName aliasName;
- private final TypeName typeName;
private final int batchSize;
public ElasticSearchIndexer(RestHighLevelClient client,
- WriteAliasName aliasName,
- TypeName typeName) {
- this(client, aliasName, typeName, DEFAULT_BATCH_SIZE);
+ WriteAliasName aliasName) {
+ this(client, aliasName, DEFAULT_BATCH_SIZE);
}
@VisibleForTesting
public ElasticSearchIndexer(RestHighLevelClient client,
WriteAliasName aliasName,
- TypeName typeName,
int batchSize) {
this.client = client;
this.aliasName = aliasName;
- this.typeName = typeName;
this.batchSize = batchSize;
}
@@ -77,7 +73,9 @@ public class ElasticSearchIndexer {
LOGGER.debug("Indexing {}: {}", id, StringUtils.left(content, DEBUG_MAX_LENGTH_CONTENT));
}
return client.index(
- new IndexRequest(aliasName.getValue(), typeName.getValue(), id)
+ new IndexRequest(aliasName.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id)
.source(content, XContentType.JSON),
RequestOptions.DEFAULT);
}
@@ -88,7 +86,7 @@ public class ElasticSearchIndexer {
BulkRequest request = new BulkRequest();
updatedDocumentParts.forEach(updatedDocumentPart -> request.add(
new UpdateRequest(aliasName.getValue(),
- typeName.getValue(),
+ NodeMappingFactory.DEFAULT_MAPPING_NAME,
updatedDocumentPart.getId())
.doc(updatedDocumentPart.getUpdatedDocumentPart(), XContentType.JSON)));
return Optional.of(client.bulk(request, RequestOptions.DEFAULT));
@@ -102,10 +100,9 @@ public class ElasticSearchIndexer {
try {
BulkRequest request = new BulkRequest();
ids.forEach(id -> request.add(
- new DeleteRequest(
- aliasName.getValue(),
- typeName.getValue(),
- id)));
+ new DeleteRequest(aliasName.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id)));
return Optional.of(client.bulk(request, RequestOptions.DEFAULT));
} catch (ValidationException e) {
LOGGER.warn("Error while deleting index", e);
@@ -115,7 +112,7 @@ public class ElasticSearchIndexer {
public void deleteAllMatchingQuery(QueryBuilder queryBuilder) {
DeleteByQueryRequest request = new DeleteByQueryRequest(aliasName.getValue())
- .setDocTypes(typeName.getValue())
+ .setDocTypes(NodeMappingFactory.DEFAULT_MAPPING_NAME)
.setScroll(TIMEOUT)
.setQuery(queryBuilder)
.setBatchSize(batchSize);
diff --git a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java
index eda3c50..237f7fa 100644
--- a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java
+++ b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/NodeMappingFactory.java
@@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
public class NodeMappingFactory {
+ public static final String DEFAULT_MAPPING_NAME = "_doc";
public static final String BOOLEAN = "boolean";
public static final String TYPE = "type";
public static final String LONG = "long";
@@ -38,6 +39,7 @@ public class NodeMappingFactory {
public static final String NOT_ANALYZED = "not_analyzed";
public static final String STRING = "string";
public static final String TEXT = "text";
+ public static final String KEYWORD = "keyword";
public static final String PROPERTIES = "properties";
public static final String DATE = "date";
public static final String FORMAT = "format";
@@ -50,14 +52,14 @@ public class NodeMappingFactory {
public static final String SNOWBALL = "snowball";
public static final String IGNORE_ABOVE = "ignore_above";
- public static RestHighLevelClient applyMapping(RestHighLevelClient client, IndexName indexName, TypeName typeName, XContentBuilder mappingsSources) throws IOException {
- if (!mappingAlreadyExist(client, indexName, typeName)) {
+ public static RestHighLevelClient applyMapping(RestHighLevelClient client, IndexName indexName, XContentBuilder mappingsSources) throws IOException {
+ if (!mappingAlreadyExist(client, indexName)) {
createMapping(client, indexName, mappingsSources);
}
return client;
}
- public static boolean mappingAlreadyExist(RestHighLevelClient client, IndexName indexName, TypeName typeName) throws IOException {
+ public static boolean mappingAlreadyExist(RestHighLevelClient client, IndexName indexName) throws IOException {
return Iterators.toStream(client.indices()
.getMapping(
new GetMappingsRequest()
@@ -66,12 +68,10 @@ public class NodeMappingFactory {
.mappings()
.values()
.iterator())
- .anyMatch(mapping -> mapping.type().contains(typeName.getValue()));
+ .anyMatch(mappingMetaData -> !mappingMetaData.getSourceAsMap().isEmpty());
}
public static void createMapping(RestHighLevelClient client, IndexName indexName, XContentBuilder mappingsSources) throws IOException {
- PutMappingRequest request = new PutMappingRequest(indexName.getValue())
- .source(mappingsSources);
client.indices().putMapping(
new PutMappingRequest(indexName.getValue())
.source(mappingsSources),
diff --git a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/TypeName.java b/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/TypeName.java
deleted file mode 100644
index 7f3dbf7..0000000
--- a/backends-common/elasticsearch-v6/src/main/java/org/apache/james/backends/es/v6/TypeName.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.backends.es.v6;
-
-public class TypeName {
- private final String value;
-
- public TypeName(String value) {
- this.value = value;
- }
-
- public String getValue() {
- return value;
- }
-}
diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java
index e945e31..08d891b 100644
--- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java
+++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/DockerElasticSearchRule.java
@@ -42,4 +42,8 @@ public class DockerElasticSearchRule extends ExternalResource {
public void awaitForElasticSearch() {
dockerElasticSearch.awaitForElasticSearch();
}
+
+ public DockerElasticSearch getDockerElasticSearch() {
+ return dockerElasticSearch;
+ }
}
diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java
index 08f02a2..3ca61c1 100644
--- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java
+++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/ElasticSearchIndexerTest.java
@@ -42,7 +42,6 @@ public class ElasticSearchIndexerTest {
private static final int MINIMUM_BATCH_SIZE = 1;
private static final IndexName INDEX_NAME = new IndexName("index_name");
private static final WriteAliasName ALIAS_NAME = new WriteAliasName("alias_name");
- private static final TypeName TYPE_NAME = new TypeName("type_name");
@Rule
public DockerElasticSearchRule elasticSearch = new DockerElasticSearchRule();
@@ -54,7 +53,7 @@ public class ElasticSearchIndexerTest {
.useIndex(INDEX_NAME)
.addAlias(ALIAS_NAME)
.createIndexAndAliases(getESClient());
- testee = new ElasticSearchIndexer(getESClient(), ALIAS_NAME, TYPE_NAME, MINIMUM_BATCH_SIZE);
+ testee = new ElasticSearchIndexer(getESClient(), ALIAS_NAME, MINIMUM_BATCH_SIZE);
}
private RestHighLevelClient getESClient() {
@@ -72,7 +71,6 @@ public class ElasticSearchIndexerTest {
try (RestHighLevelClient client = getESClient()) {
SearchResponse searchResponse = client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "trying"))),
RequestOptions.DEFAULT);
assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1);
@@ -99,7 +97,6 @@ public class ElasticSearchIndexerTest {
try (RestHighLevelClient client = getESClient()) {
SearchResponse searchResponse = client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("message", "mastering"))),
RequestOptions.DEFAULT);
assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1);
@@ -108,7 +105,6 @@ public class ElasticSearchIndexerTest {
try (RestHighLevelClient client = getESClient()) {
SearchResponse searchResponse = client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchQuery("field", "unchanged"))),
RequestOptions.DEFAULT);
assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1);
@@ -154,7 +150,6 @@ public class ElasticSearchIndexerTest {
await().atMost(Duration.TEN_SECONDS)
.until(() -> client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())),
RequestOptions.DEFAULT)
.getHits().getTotalHits() == 0);
@@ -186,7 +181,6 @@ public class ElasticSearchIndexerTest {
await().atMost(Duration.TEN_SECONDS)
.until(() -> client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())),
RequestOptions.DEFAULT)
.getHits().getTotalHits() == 1);
@@ -207,7 +201,6 @@ public class ElasticSearchIndexerTest {
try (RestHighLevelClient client = getESClient()) {
SearchResponse searchResponse = client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())),
RequestOptions.DEFAULT);
assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(0);
@@ -238,7 +231,6 @@ public class ElasticSearchIndexerTest {
try (RestHighLevelClient client = getESClient()) {
SearchResponse searchResponse = client.search(
new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())),
RequestOptions.DEFAULT);
assertThat(searchResponse.getHits().getTotalHits()).isEqualTo(1);
diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java
index 95207ff..86fad09 100644
--- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java
+++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/NodeMappingFactoryTest.java
@@ -19,10 +19,9 @@
package org.apache.james.backends.es.v6;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.assertThatCode;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.junit.Before;
import org.junit.Rule;
@@ -32,7 +31,6 @@ public class NodeMappingFactoryTest {
private static final String MESSAGE = "message";
private static final IndexName INDEX_NAME = new IndexName("index");
private static final ReadAliasName ALIAS_NAME = new ReadAliasName("alias");
- private static final TypeName TYPE_NAME = new TypeName("type");
@Rule
public DockerElasticSearchRule elasticSearch = new DockerElasticSearchRule();
@@ -47,7 +45,6 @@ public class NodeMappingFactoryTest {
.createIndexAndAliases(clientProvider.get());
NodeMappingFactory.applyMapping(clientProvider.get(),
INDEX_NAME,
- TYPE_NAME,
getMappingsSources());
}
@@ -55,24 +52,21 @@ public class NodeMappingFactoryTest {
public void applyMappingShouldNotThrowWhenCalledSeveralTime() throws Exception {
NodeMappingFactory.applyMapping(clientProvider.get(),
INDEX_NAME,
- TYPE_NAME,
getMappingsSources());
}
@Test
- public void applyMappingShouldThrowWhenTryingIndexerChanges() throws Exception {
+ public void applyMappingShouldNotThrowWhenIncrementalChanges() throws Exception {
NodeMappingFactory.applyMapping(clientProvider.get(),
INDEX_NAME,
- TYPE_NAME,
getMappingsSources());
elasticSearch.awaitForElasticSearch();
- assertThatThrownBy(() ->NodeMappingFactory.applyMapping(clientProvider.get(),
+ assertThatCode(() ->NodeMappingFactory.applyMapping(clientProvider.get(),
INDEX_NAME,
- TYPE_NAME,
getOtherMappingsSources()))
- .isInstanceOf(ElasticsearchStatusException.class);
+ .doesNotThrowAnyException();
}
private XContentBuilder getMappingsSources() throws Exception {
diff --git a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java
index c9709b1..3840181 100644
--- a/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java
+++ b/backends-common/elasticsearch-v6/src/test/java/org/apache/james/backends/es/v6/search/ScrollIterableTest.java
@@ -32,8 +32,8 @@ import org.apache.james.backends.es.v6.DockerElasticSearchRule;
import org.apache.james.backends.es.v6.ElasticSearchConfiguration;
import org.apache.james.backends.es.v6.IndexCreationFactory;
import org.apache.james.backends.es.v6.IndexName;
+import org.apache.james.backends.es.v6.NodeMappingFactory;
import org.apache.james.backends.es.v6.ReadAliasName;
-import org.apache.james.backends.es.v6.TypeName;
import org.awaitility.Duration;
import org.awaitility.core.ConditionFactory;
import org.elasticsearch.action.index.IndexRequest;
@@ -55,7 +55,6 @@ public class ScrollIterableTest {
private static final String MESSAGE = "message";
private static final IndexName INDEX_NAME = new IndexName("index");
private static final ReadAliasName ALIAS_NAME = new ReadAliasName("alias");
- private static final TypeName TYPE_NAME = new TypeName("messages");
private static final ConditionFactory WAIT_CONDITION = await().timeout(Duration.FIVE_SECONDS);
@@ -77,7 +76,6 @@ public class ScrollIterableTest {
public void scrollIterableShouldWorkWhenEmpty() throws Exception {
try (RestHighLevelClient client = clientProvider.get()) {
SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.scroll(TIMEOUT)
.source(new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
@@ -92,7 +90,9 @@ public class ScrollIterableTest {
public void scrollIterableShouldWorkWhenOneElement() throws Exception {
try (RestHighLevelClient client = clientProvider.get()) {
String id = "1";
- client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id)
+ client.index(new IndexRequest(INDEX_NAME.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id)
.source(MESSAGE, "Sample message"),
RequestOptions.DEFAULT);
@@ -100,7 +100,6 @@ public class ScrollIterableTest {
WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id));
SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.scroll(TIMEOUT)
.source(new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
@@ -115,12 +114,16 @@ public class ScrollIterableTest {
public void scrollIterableShouldWorkWhenSizeElement() throws Exception {
try (RestHighLevelClient client = clientProvider.get()) {
String id1 = "1";
- client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id1)
+ client.index(new IndexRequest(INDEX_NAME.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id1)
.source(MESSAGE, "Sample message"),
RequestOptions.DEFAULT);
String id2 = "2";
- client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id2)
+ client.index(new IndexRequest(INDEX_NAME.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id2)
.source(MESSAGE, "Sample message"),
RequestOptions.DEFAULT);
@@ -128,7 +131,6 @@ public class ScrollIterableTest {
WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id1, id2));
SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.scroll(TIMEOUT)
.source(new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
@@ -143,17 +145,23 @@ public class ScrollIterableTest {
public void scrollIterableShouldWorkWhenMoreThanSizeElement() throws Exception {
try (RestHighLevelClient client = clientProvider.get()) {
String id1 = "1";
- client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id1)
+ client.index(new IndexRequest(INDEX_NAME.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id1)
.source(MESSAGE, "Sample message"),
RequestOptions.DEFAULT);
String id2 = "2";
- client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id2)
+ client.index(new IndexRequest(INDEX_NAME.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id2)
.source(MESSAGE, "Sample message"),
RequestOptions.DEFAULT);
String id3 = "3";
- client.index(new IndexRequest(INDEX_NAME.getValue(), TYPE_NAME.getValue(), id3)
+ client.index(new IndexRequest(INDEX_NAME.getValue())
+ .type(NodeMappingFactory.DEFAULT_MAPPING_NAME)
+ .id(id3)
.source(MESSAGE, "Sample message"),
RequestOptions.DEFAULT);
@@ -161,7 +169,6 @@ public class ScrollIterableTest {
WAIT_CONDITION.untilAsserted(() -> hasIdsInIndex(client, id1, id2, id3));
SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.scroll(TIMEOUT)
.source(new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery())
@@ -181,7 +188,6 @@ public class ScrollIterableTest {
private void hasIdsInIndex(RestHighLevelClient client, String... ids) throws IOException {
SearchRequest searchRequest = new SearchRequest(INDEX_NAME.getValue())
- .types(TYPE_NAME.getValue())
.scroll(TIMEOUT)
.source(new SearchSourceBuilder()
.query(QueryBuilders.matchAllQuery()));
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org