You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2023/02/02 16:47:19 UTC
[unomi] branch monthlyToRolloverIndicesPOC updated: cleanup a bit + add support of prefix for the ILM policy registering to avoid potential conflicts
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch monthlyToRolloverIndicesPOC
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/monthlyToRolloverIndicesPOC by this push:
new 14d5731e7 cleanup a bit + add support of prefix for the ILM policy registering to avoid potential conflicts
14d5731e7 is described below
commit 14d5731e72a49cd473059867b23a1dc988b7c36a
Author: Kevan <ke...@jahia.com>
AuthorDate: Thu Feb 2 17:47:08 2023 +0100
cleanup a bit + add support of prefix for the ILM policy registering to avoid potential conflicts
---
.../main/resources/etc/custom.system.properties | 13 ++++----
.../ElasticSearchPersistenceServiceImpl.java | 39 +++++++++-------------
.../resources/OSGI-INF/blueprint/blueprint.xml | 2 +-
.../org.apache.unomi.persistence.elasticsearch.cfg | 24 +++++++------
4 files changed, 36 insertions(+), 42 deletions(-)
diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties
index e82b06544..e8b049d40 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -112,17 +112,18 @@ org.apache.unomi.elasticsearch.monthlyIndex.nbShards=${env:UNOMI_ELASTICSEARCH_M
org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_REPLICAS:-0}
org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_MAPPINGTOTALFIELDSLIMIT:-1000}
org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_MAXDOCVALUEFIELDSSEARCH:-1000}
+org.apache.unomi.elasticsearch.monthlyIndex.itemsMonthlyIndexedOverride=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_ITEMSMONTHLYINDEXED:-event,session}
+# New rollover properties (it overrides monthlyIndex values)
+org.apache.unomi.elasticsearch.rollover.nbShards=${env:UNOMI_ELASTICSEARCH_ROLLOVER_SHARDS}
+org.apache.unomi.elasticsearch.rollover.nbReplicas=${env:UNOMI_ELASTICSEARCH_ROLLOVER_REPLICAS}
+org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT}
+org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH}
+org.apache.unomi.elasticsearch.rollover.indices=${env:UNOMI_ELASTICSEARCH_ROLLOVER_INDICES}
# Rollover configuration
-org.apache.unomi.elasticsearch.rollover.indices=${env:UNOMI_ELASTICSEARCH_ROLLOVER_INDICES:-event,session}
org.apache.unomi.elasticsearch.rollover.maxSize=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXSIZE:-}
org.apache.unomi.elasticsearch.rollover.maxAge=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXAGE:-365d}
org.apache.unomi.elasticsearch.rollover.maxDocs=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXDOCS:-}
-# Rollover properties (it overrides monthlyIndex values)
-org.apache.unomi.elasticsearch.rollover.nbShards=${env:UNOMI_ELASTICSEARCH_ROLLOVER_SHARDS}
-org.apache.unomi.elasticsearch.rollover.nbReplicas=${env:UNOMI_ELASTICSEARCH_ROLLOVER_REPLICAS}
-org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT}
-org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH}
org.apache.unomi.elasticsearch.defaultIndex.nbShards=${env:UNOMI_ELASTICSEARCH_DEFAULTINDEX_SHARDS:-5}
org.apache.unomi.elasticsearch.defaultIndex.nbReplicas=${env:UNOMI_ELASTICSEARCH_DEFAULTINDEX_REPLICAS:-0}
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 8de571eb4..c06732209 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -175,20 +175,12 @@ import static org.elasticsearch.index.query.QueryBuilders.termQuery;
@SuppressWarnings("rawtypes")
public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SynchronousBundleListener {
- public static final String NUMBER_OF_SHARDS = "number_of_shards";
- public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
- public static final String CLUSTER_NAME = "cluster.name";
- public static final String BULK_PROCESSOR_NAME = "bulkProcessor.name";
public static final String BULK_PROCESSOR_CONCURRENT_REQUESTS = "bulkProcessor.concurrentRequests";
public static final String BULK_PROCESSOR_BULK_ACTIONS = "bulkProcessor.bulkActions";
public static final String BULK_PROCESSOR_BULK_SIZE = "bulkProcessor.bulkSize";
public static final String BULK_PROCESSOR_FLUSH_INTERVAL = "bulkProcessor.flushInterval";
public static final String BULK_PROCESSOR_BACKOFF_POLICY = "bulkProcessor.backoffPolicy";
public static final String MONTHLY_INDEX_ITEMS_MONTHLY_INDEXED = "monthlyIndex.itemsMonthlyIndexedOverride";
- public static final String ROLLOVER_INDICES = "rollover.indicesOverride";
- public static final String ROLLOVER_MAX_SIZE = "rollover.maxSize";
- public static final String ROLLOVER_MAX_AGE = "rollover.maxAge";
- public static final String ROLLOVER_MAX_DOCS = "rollover.maxDocs";
public static final String INDEX_DATE_PREFIX = "date-";
public static final String SEQ_NO = "seq_no";
public static final String PRIMARY_TERM = "primary_term";
@@ -215,7 +207,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private Map<String, String> mappings = new HashMap<String, String>();
private ConditionEvaluatorDispatcher conditionEvaluatorDispatcher;
private ConditionESQueryBuilderDispatcher conditionESQueryBuilderDispatcher;
- private List<String> rolloverIndices;
+ private List<String> itemsMonthlyIndexed;
private Map<String, String> routingByType;
private Integer defaultQueryLimit = 10;
@@ -229,10 +221,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String bulkProcessorBackoffPolicy = "exponential";
// Rollover configuration
+ private List<String> rolloverIndices;
private String rolloverMaxSize;
private String rolloverMaxAge;
private String rolloverMaxDocs;
- private String rolloverIndicesOverride;
private String rolloverIndexNumberOfShards;
private String rolloverIndexNumberOfReplicas;
private String rolloverIndexMappingTotalFieldsLimit;
@@ -378,8 +370,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
this.bulkProcessorBackoffPolicy = bulkProcessorBackoffPolicy;
}
- public void setRolloverIndicesOverride(String rolloverIndicesOverride) {
- this.rolloverIndicesOverride = rolloverIndicesOverride;
+ public void setRolloverIndices(String rolloverIndices) {
+ this.rolloverIndices = StringUtils.isNotEmpty(rolloverIndices) ? Arrays.asList(rolloverIndices.split(",").clone()) : null;
}
public void setRolloverMaxSize(String rolloverMaxSize) {
@@ -487,18 +479,13 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
new InClassLoaderExecute<Object>(null, null, this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) {
public Object execute(Object... args) throws Exception {
+ // Deprecated will be removed in Unomi 3.0.0
bulkProcessorConcurrentRequests = System.getProperty(BULK_PROCESSOR_CONCURRENT_REQUESTS, bulkProcessorConcurrentRequests);
bulkProcessorBulkActions = System.getProperty(BULK_PROCESSOR_BULK_ACTIONS, bulkProcessorBulkActions);
bulkProcessorBulkSize = System.getProperty(BULK_PROCESSOR_BULK_SIZE, bulkProcessorBulkSize);
bulkProcessorFlushInterval = System.getProperty(BULK_PROCESSOR_FLUSH_INTERVAL, bulkProcessorFlushInterval);
bulkProcessorBackoffPolicy = System.getProperty(BULK_PROCESSOR_BACKOFF_POLICY, bulkProcessorBackoffPolicy);
- rolloverMaxAge = System.getProperty(ROLLOVER_MAX_AGE, rolloverMaxAge);
- rolloverMaxSize = System.getProperty(ROLLOVER_MAX_SIZE, rolloverMaxSize);
- rolloverMaxDocs = System.getProperty(ROLLOVER_MAX_DOCS, rolloverMaxDocs);
- // Set rollover indices
- final String rollOverIndicesEnvValue = StringUtils.defaultIfEmpty(System.getProperty(ROLLOVER_INDICES), System.getProperty(MONTHLY_INDEX_ITEMS_MONTHLY_INDEXED));
- final String rollOverIndicesConfValue = StringUtils.defaultIfEmpty(rolloverIndicesOverride, itemsMonthlyIndexedOverride);
- rolloverIndices = rollOverIndicesConfValue.equals("none") ? Collections.emptyList() : Arrays.asList(StringUtils.defaultIfEmpty(rollOverIndicesEnvValue, rollOverIndicesConfValue).split(",").clone());
+ itemsMonthlyIndexed = itemsMonthlyIndexedOverride.equals("none") ? Collections.emptyList() : Arrays.asList(System.getProperty(MONTHLY_INDEX_ITEMS_MONTHLY_INDEXED, itemsMonthlyIndexedOverride).split(",").clone());
buildClient();
@@ -835,7 +822,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
itemType = customItemType;
}
- if (rolloverIndices.contains(itemType)) {
+ if (isItemTypeRollingOver(itemType)) {
return new MetricAdapter<T>(metricsService, ".loadItemWithQuery") {
@Override
public T execute(Object... args) throws Exception {
@@ -1395,7 +1382,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
Map<String, LifecycleAction> deleteActions = Collections.singletonMap(DeleteAction.NAME, new DeleteAction());
phases.put("delete", new Phase("delete", new TimeValue(90, TimeUnit.DAYS), deleteActions));
- LifecyclePolicy policy = new LifecyclePolicy(ROLLOVER_LIFECYCLE_NAME, phases);
+ LifecyclePolicy policy = new LifecyclePolicy(indexPrefix + "-" + ROLLOVER_LIFECYCLE_NAME, phases);
PutLifecyclePolicyRequest request = new PutLifecyclePolicyRequest(policy);
org.elasticsearch.client.core.AcknowledgedResponse putLifecyclePolicy = client.indexLifecycle().putLifecyclePolicy(request, RequestOptions.DEFAULT);
return putLifecyclePolicy.isAcknowledged();
@@ -1417,7 +1404,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
boolean indexExists = client.indices().exists(getIndexRequest, RequestOptions.DEFAULT);
if (!indexExists) {
- if (rolloverIndices.contains(itemType)) {
+ if (isItemTypeRollingOver(itemType)) {
internalCreateRolloverTemplate(itemType);
internalCreateRolloverIndex(index);
} else {
@@ -1469,7 +1456,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
" \"number_of_replicas\" : " + StringUtils.defaultIfEmpty(rolloverIndexNumberOfReplicas, monthlyIndexNumberOfReplicas) + ",\n" +
" \"mapping.total_fields.limit\" : " + StringUtils.defaultIfEmpty(rolloverIndexMappingTotalFieldsLimit, monthlyIndexMappingTotalFieldsLimit) + ",\n" +
" \"max_docvalue_fields_search\" : " + StringUtils.defaultIfEmpty(rolloverIndexMaxDocValueFieldsSearch, monthlyIndexMaxDocValueFieldsSearch) + ",\n" +
- " \"lifecycle.name\": \"" + ROLLOVER_LIFECYCLE_NAME + "\",\n" +
+ " \"lifecycle.name\": \"" + (indexPrefix + "-" + ROLLOVER_LIFECYCLE_NAME) + "\",\n" +
" \"lifecycle.rollover_alias\": \"" + rolloverAlias + "\"" +
"" +
" },\n" +
@@ -2608,7 +2595,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
private String getIndexNameForQuery(String itemType) {
- return rolloverIndices.contains(itemType) ? getRolloverIndexForQuery(itemType) : getIndex(itemType);
+ return isItemTypeRollingOver(itemType) ? getRolloverIndexForQuery(itemType) : getIndex(itemType);
}
private String getRolloverIndexForQuery(String itemType) {
@@ -2619,6 +2606,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
return (indexPrefix + "-" + indexItemTypePart).toLowerCase();
}
+ private boolean isItemTypeRollingOver(String itemType) {
+ return (rolloverIndices != null ? rolloverIndices : itemsMonthlyIndexed).contains(itemType);
+ }
+
private WriteRequest.RefreshPolicy getRefreshPolicy(String itemType) {
if (itemTypeToRefreshPolicy.containsKey(itemType)) {
return itemTypeToRefreshPolicy.get(itemType);
diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index 9b9638e94..83d4ecd41 100644
--- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -133,7 +133,7 @@
<property name="bulkProcessorFlushInterval" value="${es.bulkProcessor.flushInterval}" />
<property name="bulkProcessorBackoffPolicy" value="${es.bulkProcessor.backoffPolicy}" />
- <property name="rolloverIndicesOverride" value="${es.rollover.indices}" />
+ <property name="rolloverIndices" value="${es.rollover.indices}" />
<property name="rolloverMaxSize" value="${es.rollover.maxSize}" />
<property name="rolloverMaxAge" value="${es.rollover.maxAge}" />
<property name="rolloverMaxDocs" value="${es.rollover.maxDocs}" />
diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
index 9bcb30966..9d055cff6 100644
--- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
+++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg
@@ -28,6 +28,13 @@ monthlyIndex.numberOfShards=${org.apache.unomi.elasticsearch.monthlyIndex.nbShar
monthlyIndex.numberOfReplicas=${org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas:-0}
monthlyIndex.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000}
monthlyIndex.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch:-1000}
+monthlyIndex.itemsMonthlyIndexedOverride=${org.apache.unomi.elasticsearch.monthlyIndex.itemsMonthlyIndexedOverride:-event,session}
+# New properties for index rotation:
+rollover.numberOfShards=${org.apache.unomi.elasticsearch.rollover.nbShards}
+rollover.numberOfReplicas=${org.apache.unomi.elasticsearch.rollover.nbReplicas}
+rollover.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit}
+rollover.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch}
+rollover.indices=${org.apache.unomi.elasticsearch.rollover.indices}
numberOfShards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5}
numberOfReplicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0}
@@ -35,6 +42,12 @@ indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.defaultIndex.index
indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.defaultIndex.indexMaxDocValueFieldsSearch:-1000}
defaultQueryLimit=${org.apache.unomi.elasticsearch.defaultQueryLimit:-10}
+# Rollover amd index configuration for event and session indices, values are cumulative
+# See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/ilm-rollover.html for option details.
+rollover.maxSize=${org.apache.unomi.elasticsearch.rollover.maxSize}
+rollover.maxAge=${org.apache.unomi.elasticsearch.rollover.maxAge:-365d}
+rollover.maxDocs=${org.apache.unomi.elasticsearch.rollover.maxDocs}
+
# The following settings control the behavior of the BulkProcessor API. You can find more information about these
# settings and their behavior here : https://www.elastic.co/guide/en/elasticsearch/client/java-api/2.4/java-docs-bulk-processor.html
# The values used here are the default values of the API
@@ -44,17 +57,6 @@ bulkProcessor.bulkSize=${org.apache.unomi.elasticsearch.bulkProcessor.bulkSize:-
bulkProcessor.flushInterval=${org.apache.unomi.elasticsearch.bulkProcessor.flushInterval:-5s}
bulkProcessor.backoffPolicy=${org.apache.unomi.elasticsearch.bulkProcessor.backoffPolicy:-exponential}
-# Rollover amd index configuration for event and session indices, values are cumulative, see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/ilm-rollover.html for option details.
-rollover.indices=${org.apache.unomi.elasticsearch.rollover.indices}
-rollover.maxSize=${org.apache.unomi.elasticsearch.rollover.maxSize}
-rollover.maxAge=${org.apache.unomi.elasticsearch.rollover.maxAge:-365d}
-rollover.maxDocs=${org.apache.unomi.elasticsearch.rollover.maxDocs}
-
-rollover.numberOfShards=${org.apache.unomi.elasticsearch.rollover.nbShards}
-rollover.numberOfReplicas=${org.apache.unomi.elasticsearch.rollover.nbReplicas}
-rollover.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit}
-rollover.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch}
-
# The following settings are used to perform version checks on the connected ElasticSearch cluster, to make sure that
# appropriate versions are used. The check is performed like this :
# for each node in the ElasticSearch cluster: