You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2023/02/14 16:09:45 UTC
[unomi] branch master updated: Unomi 728 index migration (#575)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new f3c44536a Unomi 728 index migration (#575)
f3c44536a is described below
commit f3c44536a143b9e17ddeb7260937ce5046af6602
Author: jsinovassin <58...@users.noreply.github.com>
AuthorDate: Tue Feb 14 17:09:37 2023 +0100
Unomi 728 index migration (#575)
* UNOMI-728 : add migration scripts to 2.2.0 version
* UNOMI-728 : add integrations tests
---
.../test/java/org/apache/unomi/itests/AllITs.java | 4 +-
...grate16xTo200IT.java => Migrate16xTo220IT.java} | 52 ++++++++++-
.../migration/match_all_body_request.json | 5 +
.../must_not_match_some_eventype_body.json | 18 ++++
.../resources/migration/snapshots_repository.zip | Bin 868699 -> 3886208 bytes
.../shell/migration/service/MigrationConfig.java | 8 +-
.../shell/migration/utils/MigrationUtils.java | 82 ++++++++++++----
...-2.2.0-00-rolloverAndMigrateEventSession.groovy | 104 +++++++++++++++++++++
.../main/resources/org.apache.unomi.migration.cfg | 3 +
.../2.2.0/base_index_withRollover_request.json | 30 ++++++
.../requestBody/2.2.0/base_reindex_request.json | 8 ++
.../2.2.0/create_rollover_policy_query.json | 19 ++++
.../requestBody/2.2.0/match_all_body_request.json | 5 +
.../2.2.0/update_settings_poll_interval.json | 5 +
14 files changed, 321 insertions(+), 22 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index a968eaa57..60502de03 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -17,7 +17,7 @@
package org.apache.unomi.itests;
-import org.apache.unomi.itests.migration.Migrate16xTo200IT;
+import org.apache.unomi.itests.migration.Migrate16xTo220IT;
import org.apache.unomi.itests.graphql.*;
import org.apache.unomi.itests.migration.MigrationIT;
import org.junit.runner.RunWith;
@@ -31,7 +31,7 @@ import org.junit.runners.Suite.SuiteClasses;
*/
@RunWith(Suite.class)
@SuiteClasses({
- Migrate16xTo200IT.class,
+ Migrate16xTo220IT.class,
MigrationIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
similarity index 83%
rename from itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
rename to itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
index 5bf809d11..a3e320a3c 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
@@ -16,6 +16,7 @@
*/
package org.apache.unomi.itests.migration;
+import com.fasterxml.jackson.databind.JsonNode;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.unomi.api.*;
import org.apache.unomi.itests.BaseIT;
@@ -33,8 +34,12 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
-public class Migrate16xTo200IT extends BaseIT {
+public class Migrate16xTo220IT extends BaseIT {
+ private int eventCount = 0;
+ private int sessionCount = 0;
+
+ private static final int NUMBER_DUPLICATE_SESSIONS = 3;
@Override
@Before
public void waitForStartup() throws InterruptedException {
@@ -49,7 +54,9 @@ public class Migrate16xTo200IT extends BaseIT {
throw new RuntimeException("Unable to retrieve 1.6.x snapshot for ES restore");
}
// Restore the snapshot
- HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true", "{}", null);
+ HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true", "{}", null);
+ fillNumberEventAndSessionBeforeMigration(httpClient);
+
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -84,6 +91,29 @@ public class Migrate16xTo200IT extends BaseIT {
checkViewEventRestructured();
checkEventTypesNotPersistedAnymore();
checkForMappingUpdates();
+ checkEventSessionRollover2_2_0();
+ }
+
+ /**
+ * Checks if at least the new index for events and sessions exists.
+ */
+ private void checkEventSessionRollover2_2_0() throws IOException {
+ Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400", "context-event-000001"));
+ Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400", "context-session-000001"));
+
+ int newEventcount = 0;
+ for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-event-0")) {
+ JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null));
+ newEventcount += jsonNode.get("count").asInt();
+ }
+
+ int newSessioncount = 0;
+ for (String sessionIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-session-0")) {
+ JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + sessionIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null));
+ newSessioncount += jsonNode.get("count").asInt();
+ }
+ Assert.assertEquals(eventCount, newEventcount);
+ Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS, newSessioncount);
}
/**
@@ -266,4 +296,22 @@ public class Migrate16xTo200IT extends BaseIT {
Assert.assertNotNull(persistenceService.load(masterProfile, Profile.class));
Assert.assertNull(persistenceService.load(masterProfile, ProfileAlias.class));
}
+
+ private void fillNumberEventAndSessionBeforeMigration(CloseableHttpClient httpClient) {
+ try {
+ for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-event-date")) {
+ JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_count", resourceAsString("migration/must_not_match_some_eventype_body.json"), null));
+ eventCount += jsonNode.get("count").asInt();
+ }
+
+ for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-session-date")) {
+ JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null));
+ sessionCount += jsonNode.get("count").asInt();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ }
}
diff --git a/itests/src/test/resources/migration/match_all_body_request.json b/itests/src/test/resources/migration/match_all_body_request.json
new file mode 100644
index 000000000..487927ea1
--- /dev/null
+++ b/itests/src/test/resources/migration/match_all_body_request.json
@@ -0,0 +1,5 @@
+{
+ "query": {
+ "match_all": {}
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/must_not_match_some_eventype_body.json b/itests/src/test/resources/migration/must_not_match_some_eventype_body.json
new file mode 100644
index 000000000..7fc953198
--- /dev/null
+++ b/itests/src/test/resources/migration/must_not_match_some_eventype_body.json
@@ -0,0 +1,18 @@
+{
+ "query": {
+ "bool": {
+ "must_not": [
+ {
+ "match": {
+ "eventType": "sessionCreated"
+ }
+ },
+ {
+ "match": {
+ "eventType": "updateProperties"
+ }
+ }
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/snapshots_repository.zip b/itests/src/test/resources/migration/snapshots_repository.zip
index 1252d50a1..38a4e9bfe 100644
Binary files a/itests/src/test/resources/migration/snapshots_repository.zip and b/itests/src/test/resources/migration/snapshots_repository.zip differ
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
index 16fb6728a..3d4ccd4c7 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
@@ -46,7 +46,9 @@ public class MigrationConfig {
public static final String MONTHLY_TOTAL_FIELDS_LIMIT = "monthlyIndex." + TOTAL_FIELDS_LIMIT;
public static final String MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH = "monthlyIndex." + MAX_DOC_VALUE_FIELDS_SEARCH;
public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
-
+ public static final String ROLLOVER_MAX_AGE = "rolloverMaxAge";
+ public static final String ROLLOVER_MAX_SIZE = "rolloverMaxSize";
+ public static final String ROLLOVER_MAX_DOCS = "rolloverMaxDocs";
protected static final Map<String, MigrationConfigProperty> configProperties;
static {
Map<String, MigrationConfigProperty> m = new HashMap<>();
@@ -65,6 +67,10 @@ public class MigrationConfig {
m.put(MONTHLY_TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
m.put(MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
m.put(MIGRATION_HISTORY_RECOVER, new MigrationConfigProperty("We found an existing migration attempt, should we restart from it ? (this will avoid redoing steps already completed successfully) (yes/no)", null));
+ m.put(ROLLOVER_MAX_AGE, new MigrationConfigProperty("Enter ElasticSearch index rollover configuration: max_age (default: 365d): ", "365d"));
+ m.put(ROLLOVER_MAX_SIZE, new MigrationConfigProperty("Enter ElasticSearch index rollover configuration: max_size (default: null): ", null));
+ m.put(ROLLOVER_MAX_DOCS, new MigrationConfigProperty("Enter ElasticSearch index rollover configuration: max_docs (default: null): ", null));
+
configProperties = Collections.unmodifiableMap(m);
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 40fd9f549..a12fd6ec7 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -32,9 +32,7 @@ import org.osgi.framework.BundleContext;
import java.io.*;
import java.net.URL;
import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-import java.util.Enumeration;
-import java.util.Set;
+import java.util.*;
import java.util.stream.Collectors;
import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
@@ -77,8 +75,9 @@ public class MigrationUtils {
String line;
StringBuilder value = new StringBuilder();
while ((line = br.readLine()) != null) {
- if (!line.startsWith("/*") && !line.startsWith(" *") && !line.startsWith("*/"))
+ if (!line.startsWith("/*") && !line.startsWith(" *") && !line.startsWith("*/")) {
value.append(line);
+ }
}
in.close();
return value.toString();
@@ -98,14 +97,32 @@ public class MigrationUtils {
try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
- return indexesAsJson.keySet().stream().
- filter(alias -> alias.startsWith(prefix)).
- collect(Collectors.toSet());
+ return indexesAsJson.keySet().stream().filter(alias -> alias.startsWith(prefix)).collect(Collectors.toSet());
}
}
return Collections.emptySet();
}
+ public static void cleanAllIndexWithRollover(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String prefix, String indexName) throws IOException {
+ Set<String> indexes = getIndexesPrefixedBy(httpClient, esAddress, prefix + "-" + indexName + "-000");
+ List<String> sortedIndexes = new ArrayList<>(indexes);
+ Collections.sort(sortedIndexes);
+
+ if (!sortedIndexes.isEmpty()) {
+ String lastIndexName = sortedIndexes.remove(sortedIndexes.size() - 1);
+ sortedIndexes.forEach(index -> {
+ try {
+ deleteIndex(httpClient, esAddress, index);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ String matchAllBodyRequest = resourceAsString(bundleContext, "requestBody/2.2.0/match_all_body_request.json");
+
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + lastIndexName + "/_delete_by_query", matchAllBodyRequest, null);
+ }
+ }
+
public static String extractMappingFromBundles(BundleContext bundleContext, String fileName) throws IOException {
for (Bundle bundle : bundleContext.getBundles()) {
Enumeration<URL> predefinedMappings = bundle.findEntries("META-INF/cxs/mappings", fileName, true);
@@ -131,8 +148,44 @@ public class MigrationUtils {
return settings.replace("#mappings", mapping);
}
- public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
- String newIndexSettings, String painlessScript, MigrationContext migrationContext) throws Exception {
+ public static String buildIndexCreationRequestWithRollover(String baseIndexSettings, String mapping, MigrationContext context, String lifeCycleName, String rolloverAlias) throws IOException {
+ return buildIndexCreationRequest(baseIndexSettings, mapping, context, false)
+ .replace("#lifecycleName", lifeCycleName)
+ .replace("#lifecycleRolloverAlias", rolloverAlias);
+ }
+
+ public static String buildRolloverPolicyCreationRequest(String baseRequest, MigrationContext migrationContext) throws IOException {
+
+ StringJoiner rolloverHotActions = new StringJoiner(", ");
+
+ String rolloverMaxAge = migrationContext.getConfigString("rolloverMaxAge");
+ String rolloverMaxSize = migrationContext.getConfigString("rolloverMaxSize");
+ String rolloverMaxDocs = migrationContext.getConfigString("rolloverMaxDocs");
+ if (StringUtils.isNotBlank(rolloverMaxAge)) {
+ rolloverHotActions.add("\"max_age\": \"" + rolloverMaxAge + "\"");
+ }
+ if (StringUtils.isNotBlank(rolloverMaxSize)) {
+ rolloverHotActions.add("\"max_size\": \"" + rolloverMaxSize + "\"");
+ }
+ if (StringUtils.isNotBlank(rolloverMaxDocs)) {
+ rolloverHotActions.add("\"max_docs\": \"" + rolloverMaxDocs + "\"");
+ }
+ return baseRequest.replace("#rolloverHotActions", rolloverHotActions.toString());
+ }
+
+ public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName) throws Exception {
+ String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName);
+
+ HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
+ }
+
+ public static void deleteIndex(CloseableHttpClient httpClient, String esAddress, String indexName) throws Exception {
+ if (indexExists(httpClient, esAddress, indexName)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
+ }
+ }
+
+ public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName, String newIndexSettings, String painlessScript, MigrationContext migrationContext) throws Exception {
if (indexName.endsWith("-cloned")) {
// We should never reIndex a clone ...
return;
@@ -140,9 +193,7 @@ public class MigrationUtils {
String indexNameCloned = indexName + "-cloned";
- String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json")
- .replace("#source", indexNameCloned).replace("#dest", indexName)
- .replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : "");
+ String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json").replace("#source", indexNameCloned).replace("#dest", indexName).replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : "");
String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json");
@@ -208,10 +259,7 @@ public class MigrationUtils {
}
// scroll
- response = HttpUtils.executePostRequest(httpClient, esAddress + "/_search/scroll", "{\n" +
- " \"scroll_id\": \"" + scrollId + "\",\n" +
- " \"scroll\": \"" + scrollDuration + "\"\n" +
- "}", null);
+ response = HttpUtils.executePostRequest(httpClient, esAddress + "/_search/scroll", "{\n" + " \"scroll_id\": \"" + scrollId + "\",\n" + " \"scroll\": \"" + scrollDuration + "\"\n" + "}", null);
}
}
@@ -222,7 +270,7 @@ public class MigrationUtils {
while (true) {
final JSONObject status = new JSONObject(HttpUtils.executeGetRequest(httpClient, esAddress + "/_cluster/health?wait_for_status=yellow&timeout=60s", null));
if (!status.get("timed_out").equals("true")) {
- migrationContext.printMessage("ES Cluster status is " + status.get("status"));
+ migrationContext.printMessage("ES Cluster status is " + status.get("status"));
break;
}
migrationContext.printMessage("Waiting for ES Cluster status to be Yellow, current status is " + status.get("status"));
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
new file mode 100644
index 000000000..a04a4b097
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
@@ -0,0 +1,104 @@
+import org.apache.unomi.shell.migration.service.MigrationContext
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
+String newEventIndex = indexPrefix + "-event-000001"
+String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy"
+String rolloverEventAlias = indexPrefix + "-event"
+String newSessionIndex = indexPrefix + "-session-000001"
+String rolloverSessionAlias = indexPrefix + "-session"
+
+context.performMigrationStep("2.2.0-update-lifecyle-poll-interval", () -> {
+ String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json")
+ .replace("#pollIntervalValue", "\"2s\"")
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null)
+})
+
+context.performMigrationStep("2.2.0-create-rollover-policy", () -> {
+ String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json")
+ String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context)
+
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null)
+})
+
+context.performMigrationStep("2.2.0-create-event-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
+
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null)
+ }
+})
+
+Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-event-date-")
+List<String> eventSortedIndices = new ArrayList<>(eventIndices)
+Collections.sort(eventSortedIndices)
+
+context.performMigrationStep("2.2.0-migrate-existing-events", () -> {
+ MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "event")
+ eventSortedIndices.each { eventIndex ->
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event")
+ sleep(3000)
+ }
+})
+
+context.performMigrationStep("2.2.0-remove-old-events-indices", () -> {
+ eventSortedIndices.each { eventIndex ->
+ MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, eventIndex)
+ }
+})
+
+context.performMigrationStep("2.2.0-create-session-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newSessionIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json")
+
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverSessionAlias)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newSessionIndex, newIndexSettings, null)
+ }
+})
+
+Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-session-date-")
+List<String> sessionSortedIndices = new ArrayList<>(sessionIndices)
+Collections.sort(sessionSortedIndices)
+
+context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> {
+ MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "session")
+ sessionSortedIndices.each { sessionIndex ->
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session")
+ sleep(3000)
+ }
+})
+
+context.performMigrationStep("2.2.0-remove-old-sessions-indices", () -> {
+ sessionSortedIndices.each { sessionIndex ->
+ MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, sessionIndex)
+ }
+})
+
+context.performMigrationStep("2.2.0-reset-poll-interval", () -> {
+ String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json")
+ .replace("#pollIntervalValue", "null")
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null)
+})
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index f4cd52771..cc907d93f 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -32,6 +32,9 @@ number_of_shards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5}
number_of_replicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0}
mapping.total_fields.limit=${org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit:-1000}
max_docvalue_fields_search=${org.apache.unomi.elasticsearch.defaultIndex.indexMaxDocValueFieldsSearch:-1000}
+rolloverMaxSize=${org.apache.unomi.elasticsearch.rollover.maxSize:-}
+rolloverMaxAge=${org.apache.unomi.elasticsearch.rollover.maxAge:-365d}
+rolloverMaxDocs=${org.apache.unomi.elasticsearch.rollover.maxDocs:-}
# Should the migration try to recover from a previous run ?
# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json
new file mode 100644
index 000000000..c59422642
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json
@@ -0,0 +1,30 @@
+{
+ "settings": {
+ "index": {
+ "number_of_shards": #numberOfShards,
+ "number_of_replicas": #numberOfReplicas,
+ "mapping.total_fields.limit": #mappingTotalFieldsLimit,
+ "max_docvalue_fields_search": #maxDocValueFieldsSearch,
+ "lifecycle.name": "#lifecycleName",
+ "lifecycle.rollover_alias": "#lifecycleRolloverAlias"
+ },
+ "analysis": {
+ "analyzer": {
+ "folding": {
+ "type": "custom",
+ "tokenizer": "keyword",
+ "filter": [
+ "lowercase",
+ "asciifolding"
+ ]
+ }
+ }
+ }
+ },
+ "aliases": {
+ "#lifecycleRolloverAlias": {
+ "is_write_index": true
+ }
+ },
+ "mappings": #mappings
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
new file mode 100644
index 000000000..ddcb79a5e
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
@@ -0,0 +1,8 @@
+{
+ "source": {
+ "index": "#source"
+ },
+ "dest": {
+ "index": "#dest"
+ }
+}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/create_rollover_policy_query.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/create_rollover_policy_query.json
new file mode 100644
index 000000000..c9bc94a29
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/create_rollover_policy_query.json
@@ -0,0 +1,19 @@
+{
+ "policy": {
+ "phases": {
+ "hot": {
+ "actions": {
+ "rollover": {
+ #rolloverHotActions
+ }
+ }
+ },
+ "delete": {
+ "min_age": "90d",
+ "actions": {
+ "delete": {}
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/match_all_body_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/match_all_body_request.json
new file mode 100644
index 000000000..487927ea1
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/match_all_body_request.json
@@ -0,0 +1,5 @@
+{
+ "query": {
+ "match_all": {}
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/update_settings_poll_interval.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/update_settings_poll_interval.json
new file mode 100644
index 000000000..75695e0fe
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/update_settings_poll_interval.json
@@ -0,0 +1,5 @@
+{
+ "persistent": {
+ "indices.lifecycle.poll_interval": #pollIntervalValue
+ }
+}
\ No newline at end of file