You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2020/03/10 09:34:10 UTC
[unomi] branch UNOMI-284-configurable-mappingtotalfields created
(now df71700)
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a change to branch UNOMI-284-configurable-mappingtotalfields
in repository https://gitbox.apache.org/repos/asf/unomi.git.
at df71700 UNOMI-284 Make mapping total fields limit configurable For some usages the default limit (1000) is too low. This modification makes it configurable, especially for dynamically created indices.
This branch includes the following new commits:
new df71700 UNOMI-284 Make mapping total fields limit configurable For some usages the default limit (1000) is too low. This modification makes it configurable, especially for dynamically created indices.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-284 Make mapping total fields limit
configurable For some usages the default limit (1000) is too low. This
modification makes it configurable,
especially for dynamically created indices.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch UNOMI-284-configurable-mappingtotalfields
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit df71700c94f71dddb78b4dc21d5288b7cf69be73
Author: Serge Huber <sh...@apache.org>
AuthorDate: Tue Mar 10 10:34:04 2020 +0100
UNOMI-284 Make mapping total fields limit configurable
For some usages the default limit (1000) is too low. This modification makes it configurable, especially for dynamically created indices.
---
package/src/main/resources/etc/custom.system.properties | 2 ++
.../elasticsearch/ElasticSearchPersistenceServiceImpl.java | 11 +++++++++++
.../core/src/main/resources/OSGI-INF/blueprint/blueprint.xml | 4 ++++
.../resources/org.apache.unomi.persistence.elasticsearch.cfg | 2 ++
4 files changed, 19 insertions(+)
diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties
index 1b33486..9e91a96 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -69,8 +69,10 @@ org.apache.unomi.elasticsearch.addresses=${env:UNOMI_ELASTICSEARCH_ADDRESSES:-lo
org.apache.unomi.elasticsearch.index.prefix=${env:UNOMI_ELASTICSEARCH_INDEXPREFIX:-context}
org.apache.unomi.elasticsearch.monthlyIndex.nbShards=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_SHARDS:-5}
org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_REPLICAS:-0}
+org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_MAPPINGTOTALFIELDSLIMIT:-1000}
org.apache.unomi.elasticsearch.defaultIndex.nbShards=${env:UNOMI_ELASTICSEARCH_DEFAULTINDEX_SHARDS:-5}
org.apache.unomi.elasticsearch.defaultIndex.nbReplicas=${env:UNOMI_ELASTICSEARCH_DEFAULTINDEX_REPLICAS:-0}
+org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_DEFAULTINDEX_MAPPINGTOTALFIELDSLIMIT:-1000}
org.apache.unomi.elasticsearch.defaultQueryLimit=${env:UNOMI_ELASTICSEARCH_DEFAULTQUERYLIMIT:-10}
org.apache.unomi.elasticsearch.aggregateQueryBucketSize=${env:UNOMI_ELASTICSEARCH_AGGREGATEBUCKETSIZE:-5000}
org.apache.unomi.elasticsearch.maximumIdsQueryCount=${env:UNOMI_ELASTICSEARCH_MAXIMUMIDSQUERYCOUNT:-5000}
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index b3fd181..e82241f 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -139,8 +139,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String indexPrefix;
private String monthlyIndexNumberOfShards;
private String monthlyIndexNumberOfReplicas;
+ private String monthlyIndexMappingTotalFieldsLimit;
private String numberOfShards;
private String numberOfReplicas;
+ private String indexMappingTotalFieldsLimit;
private BundleContext bundleContext;
private Map<String, String> mappings = new HashMap<String, String>();
private ConditionEvaluatorDispatcher conditionEvaluatorDispatcher;
@@ -205,6 +207,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.monthlyIndexNumberOfReplicas = monthlyIndexNumberOfReplicas;
}
+ public void setMonthlyIndexMappingTotalFieldsLimit(String monthlyIndexMappingTotalFieldsLimit) {
+ this.monthlyIndexMappingTotalFieldsLimit = monthlyIndexMappingTotalFieldsLimit;
+ }
+
public void setNumberOfShards(String numberOfShards) {
this.numberOfShards = numberOfShards;
}
@@ -213,6 +219,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.numberOfReplicas = numberOfReplicas;
}
+ public void setIndexMappingTotalFieldsLimit(String indexMappingTotalFieldsLimit) {
+ this.indexMappingTotalFieldsLimit = indexMappingTotalFieldsLimit;
+ }
+
public void setDefaultQueryLimit(Integer defaultQueryLimit) {
this.defaultQueryLimit = defaultQueryLimit;
}
@@ -972,6 +982,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
" \"index\" : {\n" +
" \"number_of_shards\" : " + monthlyIndexNumberOfShards + ",\n" +
" \"number_of_replicas\" : " + monthlyIndexNumberOfReplicas + "\n" +
+ " \"index.mapping.total_fields.limit\" : " + monthlyIndexMappingTotalFieldsLimit + "\n" +
" },\n" +
" \"analysis\": {\n" +
" \"analyzer\": {\n" +
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 8922b3e..c749aa1 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -35,8 +35,10 @@
<cm:property name="index.prefix" value="context"/>
<cm:property name="numberOfShards" value="5"/>
<cm:property name="numberOfReplicas" value="0"/>
+ <cm:property name="indexMappingTotalFieldsLimit" value="1000"/>
<cm:property name="monthlyIndex.numberOfShards" value="3"/>
<cm:property name="monthlyIndex.numberOfReplicas" value="0"/>
+ <cm:property name="monthlyIndex.indexMappingTotalFieldsLimit" value="1000"/>
<cm:property name="defaultQueryLimit" value="10"/>
<cm:property name="bulkProcessor.concurrentRequests" value="1" />
@@ -90,8 +92,10 @@
<property name="indexPrefix" value="${es.index.prefix}"/>
<property name="monthlyIndexNumberOfShards" value="${es.monthlyIndex.numberOfShards}"/>
<property name="monthlyIndexNumberOfReplicas" value="${es.monthlyIndex.numberOfReplicas}"/>
+ <property name="monthlyIndexMappingTotalFieldsLimit" value="${es.monthlyIndex.indexMappingTotalFieldsLimit}"/>
<property name="numberOfShards" value="${es.numberOfShards}"/>
<property name="numberOfReplicas" value="${es.numberOfReplicas}"/>
+ <property name="indexMappingTotalFieldsLimit" value="${es.indexMappingTotalFieldsLimit}"/>
<property name="elasticSearchAddresses" value="${es.elasticSearchAddresses}"/>
<property name="defaultQueryLimit" value="${es.defaultQueryLimit}"/>
<property name="itemsMonthlyIndexed">
diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
index 9565899..eb0e911 100644
--- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
+++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
@@ -23,8 +23,10 @@ elasticSearchAddresses=${org.apache.unomi.elasticsearch.addresses:-localhost:920
index.prefix=${org.apache.unomi.elasticsearch.index.prefix:-context}
monthlyIndex.numberOfShards=${org.apache.unomi.elasticsearch.monthlyIndex.nbShards:-5}
monthlyIndex.numberOfReplicas=${org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas:-0}
+monthlyIndex.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000}
numberOfShards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5}
numberOfReplicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0}
+indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit:-1000}
defaultQueryLimit=${org.apache.unomi.elasticsearch.defaultQueryLimit:-10}
# The following settings control the behavior of the BulkProcessor API. You can find more information about these