You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/17 14:01:52 UTC
[unomi] branch migrationAutomation updated: UNOMI-622: improve reindex by using other index settings for recovery system to work in case reindexation failure
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch migrationAutomation
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/migrationAutomation by this push:
new bf9b538be UNOMI-622: improve reindex by using other index settings for recovery system to work in case reindexation failure
bf9b538be is described below
commit bf9b538bebe3afdd46affe480fe7799e2883485a
Author: Kevan <ke...@jahia.com>
AuthorDate: Wed Aug 17 16:01:39 2022 +0200
UNOMI-622: improve reindex by using other index settings for recovery system to work in case reindexation failure
---
.../test/java/org/apache/unomi/itests/AllITs.java | 34 +---------------------
.../apache/unomi/itests/migration/MigrationIT.java | 9 ++++--
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 2 +-
.../migrate-2.0.0-05-globalReindex.groovy | 8 ++---
.../migrate-2.0.0-10-profileReindex.groovy | 2 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 4 ++-
6 files changed, 15 insertions(+), 44 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index a968eaa57..0bc104750 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -32,39 +32,7 @@ import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({
Migrate16xTo200IT.class,
- MigrationIT.class,
- BasicIT.class,
- ConditionEvaluatorIT.class,
- ConditionESQueryBuilderIT.class,
- SegmentIT.class,
- ProfileServiceIT.class,
- ProfileImportBasicIT.class,
- ProfileImportSurfersIT.class,
- ProfileImportRankingIT.class,
- ProfileImportActorsIT.class,
- ProfileExportIT.class,
- ProfileMergeIT.class,
- EventServiceIT.class,
- PropertiesUpdateActionIT.class,
- CopyPropertiesActionIT.class,
- IncrementPropertyIT.class,
- InputValidationIT.class,
- ModifyConsentIT.class,
- PatchIT.class,
- ContextServletIT.class,
- SecurityIT.class,
- RuleServiceIT.class,
- PrivacyServiceIT.class,
- GroovyActionsServiceIT.class,
- GraphQLEventIT.class,
- GraphQLListIT.class,
- GraphQLProfileIT.class,
- GraphQLProfilePropertiesIT.class,
- GraphQLSegmentIT.class,
- GraphQLWebSocketIT.class,
- JSONSchemaIT.class,
- GraphQLProfileAliasesIT.class,
- SendEventActionIT.class
+ MigrationIT.class
})
public class AllITs {
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
index a6f554c4a..a7fa068d1 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
@@ -16,8 +16,8 @@
*/
package org.apache.unomi.itests.migration;
-import graphql.Assert;
import org.apache.unomi.itests.BaseIT;
+import org.junit.Assert;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
@@ -25,6 +25,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import static org.junit.Assert.fail;
+
public class MigrationIT extends BaseIT {
protected static final Path BASE_DIRECTORIES = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts");
private static final String FAILING_SCRIPT_NAME = "migrate-11.0.0-01-failingMigration.groovy";
@@ -42,6 +44,7 @@ public class MigrationIT extends BaseIT {
Files.write(FAILING_SCRIPT_FS_PATH, bundleResourceAsString(FAILING_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
try {
executeCommand("unomi:migrate 10.0.0 true");
+ fail("Migration should have failed and crashed by Exception throwing");
} catch (Exception e) {
// this is expected, the script fail at step 3
}
@@ -53,8 +56,8 @@ public class MigrationIT extends BaseIT {
System.out.println(successResult);
// step 1 and 2 should not be contains, they passed on first attempt.
// Only step 3, 4 and 5 should be performed.
- Assert.assertTrue(!successResult.contains("inside step 1"));
- Assert.assertTrue(!successResult.contains("inside step 2"));
+ Assert.assertFalse(successResult.contains("inside step 1"));
+ Assert.assertFalse(successResult.contains("inside step 2"));
Assert.assertTrue(successResult.contains("inside step 3"));
Assert.assertTrue(successResult.contains("inside step 4"));
Assert.assertTrue(successResult.contains("inside step 5"));
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 9b38b81f6..3f46f0dd6 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -33,7 +33,7 @@ context.performMigrationStep("2.0.0-create-scope-index", () -> {
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, indexPrefix + "-profile", mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, "${indexPrefix}-profile", mapping)
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null)
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 211261a0f..826243ae4 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -25,9 +25,7 @@ String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"];
indicesToReindex.each { indexToReindex ->
- String mappingFileName = "${indexToReindex}.json"
- String realIndexName = "${indexPrefix}-${indexToReindex.toLowerCase()}"
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, mappingFileName)
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, realIndexName, mapping)
- MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, realIndexName, newIndexSettings, null, context)
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "${indexToReindex}.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-profile", mapping)
+ MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReindex.toLowerCase()}", newIndexSettings, null, context)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index 52e532bf4..4711056c0 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -24,6 +24,6 @@ String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, indexPrefix + "-profile", mapping)
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-segment", mapping)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, indexPrefix + "-profile",
newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), context)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index a98818118..941ff82a8 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -33,7 +33,9 @@ String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBod
String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-")
+// use session indices to extract monthly index settings
+Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-session-")
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, sessionIndices[0], mapping)
eventIndices.each { eventIndex ->
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, eventIndex, mapping)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context)
}
\ No newline at end of file