You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/12 09:30:12 UTC
[unomi] branch recoverMigrationSystem created (now 3e7ff9f1e)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a change to branch recoverMigrationSystem
in repository https://gitbox.apache.org/repos/asf/unomi.git
at 3e7ff9f1e UNOMI-627: migration recovery system in case of step failure
This branch includes the following new commits:
new 3e7ff9f1e UNOMI-627: migration recovery system in case of step failure
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-627: migration recovery system in case of step failure
Posted by jk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch recoverMigrationSystem
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 3e7ff9f1e0eead1b3921a2eb1afee302cfec5ac0
Author: Kevan <ke...@jahia.com>
AuthorDate: Fri Aug 12 11:29:55 2022 +0200
UNOMI-627: migration recovery system in case of step failure
---
.../test/java/org/apache/unomi/itests/AllITs.java | 2 +
.../unomi/itests/migration/Migrate16xTo200IT.java | 7 +-
.../apache/unomi/itests/migration/MigrationIT.java | 72 ++++++++++++
.../migrate-11.0.0-01-failingMigration.groovy | 39 ++++---
.../migrate-11.0.0-01-successMigration.groovy | 38 ++++---
.../migration/org.apache.unomi.migration.cfg | 5 +
tools/shell-commands/pom.xml | 17 ++-
.../apache/unomi/shell/migration/Migration.java | 2 +-
.../unomi/shell/migration/actions/Migrate.java | 27 +++--
.../migration/{ => actions}/MigrationConfig.java | 4 +-
.../{ => actions}/MigrationConfigProperty.java | 2 +-
.../shell/migration/actions/MigrationHistory.java | 124 +++++++++++++++++++++
.../migration/{ => actions}/MigrationScript.java | 4 +-
.../unomi/shell/migration/impl/MigrationTo121.java | 2 +-
.../unomi/shell/migration/impl/MigrationTo122.java | 4 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 2 +-
.../shell/migration/utils/MigrationUtils.java | 48 +++++---
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 91 +++++++++------
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 48 ++++++--
.../migrate-2.0.0-05-globalReindex.groovy | 4 +-
.../migrate-2.0.0-10-profileReindex.groovy | 4 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 11 +-
.../main/resources/org.apache.unomi.migration.cfg | 7 +-
23 files changed, 444 insertions(+), 120 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index 1db16621a..a968eaa57 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -19,6 +19,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xTo200IT;
import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.MigrationIT;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@@ -31,6 +32,7 @@ import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({
Migrate16xTo200IT.class,
+ MigrationIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
ConditionESQueryBuilderIT.class,
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
index 921ae4ab1..115182ed8 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
@@ -55,7 +55,12 @@ public class Migrate16xTo200IT extends BaseIT {
}
// Do migrate the data set
- executeCommand("unomi:migrate 1.6.0 true");
+ String commandResults = executeCommand("unomi:migrate 1.6.0 true");
+
+ // Prin the resulted output in the karaf shell directly
+ System.out.println("Migration command output results:");
+ System.out.println(commandResults);
+
// Call super for starting Unomi and wait for the complete startup
super.waitForStartup();
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
new file mode 100644
index 000000000..94f546c01
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests.migration;
+
+import graphql.Assert;
+import org.apache.unomi.itests.BaseIT;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class MigrationIT extends BaseIT {
+ protected static final Path BASE_DIRECTORIES = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts");
+ private static final String FAILING_SCRIPT_NAME = "migrate-11.0.0-01-failingMigration.groovy";
+ private static final String SUCCESS_SCRIPT_NAME = "migrate-11.0.0-01-successMigration.groovy";
+ private static final String FAILING_SCRIPT_RESOURCE = "migration/" + FAILING_SCRIPT_NAME;
+ private static final String SUCCESS_SCRIPT_RESOURCE = "migration/" + SUCCESS_SCRIPT_NAME;
+ protected static final Path FAILING_SCRIPT_FS_PATH = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts", FAILING_SCRIPT_NAME);
+ protected static final Path SUCCESS_SCRIPT_FS_PATH = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts", SUCCESS_SCRIPT_NAME);
+
+ @Test
+ public void checkMigrationRecoverySystem() throws Exception {
+ try {
+ Files.createDirectories(BASE_DIRECTORIES);
+
+ Files.write(FAILING_SCRIPT_FS_PATH, bundleResourceAsString(FAILING_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
+ String failingResult = executeCommand("unomi:migrate 10.0.0 true");
+ System.out.println("Intentional failing migration result:");
+ System.out.println(failingResult);
+ // step 4 and 5 should not be contains, step 3 is failing
+ // Only step 1, 2 and 3 should be performed.
+ Assert.assertTrue(failingResult.contains("inside step 1"));
+ Assert.assertTrue(failingResult.contains("inside step 2"));
+ Assert.assertTrue(failingResult.contains("inside step 3"));
+ Assert.assertTrue(!failingResult.contains("inside step 4"));
+ Assert.assertTrue(!failingResult.contains("inside step 5"));
+ Files.deleteIfExists(FAILING_SCRIPT_FS_PATH);
+
+ Files.write(SUCCESS_SCRIPT_FS_PATH, bundleResourceAsString(SUCCESS_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
+ String successResult = executeCommand("unomi:migrate 10.0.0 true");
+ System.out.println("Success recovered from failing migration result:");
+ System.out.println(successResult);
+ // step 1 and 2 should not be contains, they passed on first attempt.
+ // Only step 3, 4 and 5 should be performed.
+ Assert.assertTrue(!successResult.contains("inside step 1"));
+ Assert.assertTrue(!successResult.contains("inside step 2"));
+ Assert.assertTrue(successResult.contains("inside step 3"));
+ Assert.assertTrue(successResult.contains("inside step 4"));
+ Assert.assertTrue(successResult.contains("inside step 5"));
+ Files.deleteIfExists(SUCCESS_SCRIPT_FS_PATH);
+ } finally {
+ Files.deleteIfExists(FAILING_SCRIPT_FS_PATH);
+ Files.deleteIfExists(SUCCESS_SCRIPT_FS_PATH);
+ }
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
similarity index 51%
copy from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
copy to itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
index 58f4bee75..84ea5e908 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
@@ -1,3 +1,6 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -14,25 +17,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
-/**
- * Just a bean for a configuration property to be used during migration process
- */
-public class MigrationConfigProperty {
- String description;
- String defaultValue;
+MigrationHistory history = migrationHistory
+history.performMigrationStep("step 1", () -> {
+ ConsoleUtils.printMessage(session, "inside step 1")
+})
+
+history.performMigrationStep("step 2", () -> {
+ ConsoleUtils.printMessage(session, "inside step 2")
+})
- public MigrationConfigProperty(String description, String defaultValue) {
- this.description = description;
- this.defaultValue = defaultValue;
- }
+history.performMigrationStep("step 3", () -> {
+ ConsoleUtils.printMessage(session, "inside step 3")
+ throw new RuntimeException("Intentional failure !")
+})
- public String getDescription() {
- return description;
- }
+history.performMigrationStep("step 4", () -> {
+ ConsoleUtils.printMessage(session, "inside step 4")
+})
- public String getDefaultValue() {
- return defaultValue;
- }
-}
+history.performMigrationStep("step 5", () -> {
+ ConsoleUtils.printMessage(session, "inside step 5")
+})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
similarity index 53%
copy from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
copy to itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
index 58f4bee75..44c4403ad 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
@@ -1,3 +1,6 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -14,25 +17,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
-/**
- * Just a bean for a configuration property to be used during migration process
- */
-public class MigrationConfigProperty {
- String description;
- String defaultValue;
+MigrationHistory history = migrationHistory
+history.performMigrationStep("step 1", () -> {
+ ConsoleUtils.printMessage(session, "inside step 1")
+})
+
+history.performMigrationStep("step 2", () -> {
+ ConsoleUtils.printMessage(session, "inside step 2")
+})
- public MigrationConfigProperty(String description, String defaultValue) {
- this.description = description;
- this.defaultValue = defaultValue;
- }
+history.performMigrationStep("step 3", () -> {
+ ConsoleUtils.printMessage(session, "inside step 3")
+})
- public String getDescription() {
- return description;
- }
+history.performMigrationStep("step 4", () -> {
+ ConsoleUtils.printMessage(session, "inside step 4")
+})
- public String getDefaultValue() {
- return defaultValue;
- }
-}
+history.performMigrationStep("step 5", () -> {
+ ConsoleUtils.printMessage(session, "inside step 5")
+})
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg b/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
index 26cfca708..d96120c28 100644
--- a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
+++ b/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
@@ -17,7 +17,12 @@
# Migration config used for silent migration
+# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
esAddress = http://localhost:9400
esLogin =
httpClient.trustAllCertificates = true
indexPrefix = context
+
+# Should the migration try to recover from a previous run ?
+# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
+recoverFromHistory = true
\ No newline at end of file
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 1b73ab209..0a3df585f 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -93,6 +93,20 @@
<version>${groovy.version}</version>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${version.jackson.core}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${version.jackson.databind}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
@@ -103,7 +117,8 @@
<configuration>
<instructions>
<Export-Package>
- org.apache.unomi.shell.migration.utils
+ org.apache.unomi.shell.migration.utils,
+ org.apache.unomi.shell.migration.actions,
</Export-Package>
<Embed-Dependency>*;scope=compile|runtime</Embed-Dependency>
<DynamicImport-Package>*</DynamicImport-Package>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
index c7d9ece8c..0113b3854 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
@@ -18,10 +18,10 @@ package org.apache.unomi.shell.migration;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.osgi.framework.BundleContext;
import java.io.IOException;
-import java.util.Map;
/**
* @author dgaillard
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index ea5afdfde..a21b5d8c3 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -31,8 +31,6 @@ import org.apache.karaf.shell.api.action.Command;
import org.apache.karaf.shell.api.action.lifecycle.Reference;
import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.MigrationConfig;
-import org.apache.unomi.shell.migration.MigrationScript;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.osgi.framework.*;
@@ -47,13 +45,16 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.unomi.shell.migration.MigrationConfig.*;
+import static org.apache.unomi.shell.migration.actions.MigrationConfig.*;
@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " +
"It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process.")
@Service
public class Migrate implements Action {
+ protected static final String MIGRATION_FS_ROOT_FOLDER = "migration";
+ protected static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
+
@Reference
Session session;
@@ -101,6 +102,9 @@ public class Migrate implements Action {
// reset migration config from previous stored users choices.
migrationConfig.reset();
+ Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
+ MigrationHistory migrationHistory = new MigrationHistory(session, migrationConfig);
+ migrationHistory.tryRecover();
// Handle credentials
CredentialsProvider credentialsProvider = null;
@@ -115,7 +119,7 @@ public class Migrate implements Action {
try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session), credentialsProvider)) {
// Compile scripts
- scripts = parseScripts(scripts, session, httpClient, migrationConfig);
+ scripts = parseScripts(scripts, session, httpClient, migrationConfig, migrationHistory);
// Start migration
ConsoleUtils.printMessage(session, "Starting migration process from version: " + originVersion);
@@ -130,6 +134,9 @@ public class Migrate implements Action {
ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
}
+
+ // We clean history, migration is successful
+ migrationHistory.clean();
}
return null;
@@ -152,7 +159,7 @@ public class Migrate implements Action {
.collect(Collectors.toCollection(TreeSet::new));
}
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig) {
+ private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
return scripts.stream()
@@ -160,7 +167,7 @@ public class Migrate implements Action {
// fallback on current bundle if the scripts is not provided by OSGI
Bundle scriptBundle = migrateScript.getBundle() != null ? migrateScript.getBundle() : bundleContext.getBundle();
if (!shellsPerBundle.containsKey(scriptBundle.getSymbolicName())) {
- shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig));
+ shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig, migrationHistory));
}
migrateScript.setCompiledScript(shellsPerBundle.get(scriptBundle.getSymbolicName()).parse(migrateScript.getScript()));
})
@@ -186,13 +193,12 @@ public class Migrate implements Action {
private Set<MigrationScript> loadFileSystemScripts() throws IOException {
// check migration folder exists
- Path migrationFolder = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts");
- if (!Files.isDirectory(migrationFolder)) {
+ if (!Files.isDirectory(MIGRATION_FS_SCRIPTS_FOLDER)) {
return Collections.emptySet();
}
List<Path> paths;
- try (Stream<Path> walk = Files.walk(migrationFolder)) {
+ try (Stream<Path> walk = Files.walk(MIGRATION_FS_SCRIPTS_FOLDER)) {
paths = walk
.filter(path -> !Files.isDirectory(path))
.filter(path -> path.toString().toLowerCase().endsWith("groovy"))
@@ -206,13 +212,14 @@ public class Migrate implements Action {
return migrationScripts;
}
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig) {
+ private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
groovyShell.setVariable("session", session);
groovyShell.setVariable("httpClient", httpClient);
groovyShell.setVariable("migrationConfig", migrationConfig);
+ groovyShell.setVariable("migrationHistory", migrationHistory);
groovyShell.setVariable("bundleContext", bundle.getBundleContext());
return groovyShell;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
similarity index 94%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
index 41f37aac7..df01f0de0 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
+package org.apache.unomi.shell.migration.actions;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
@@ -46,6 +46,7 @@ public class MigrationConfig {
public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
public static final String TOTAL_FIELDS_LIMIT = "mapping.total_fields.limit";
public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
+ public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
private static final Map<String, MigrationConfigProperty> configProperties;
static {
@@ -59,6 +60,7 @@ public class MigrationConfig {
m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
m.put(TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
m.put(MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
+ m.put(MIGRATION_HISTORY_RECOVER, new MigrationConfigProperty("We found an existing migration attempt, should we restart from it ? (this will avoid redoing steps already completed successfully) (yes/no)", null));
configProperties = Collections.unmodifiableMap(m);
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
similarity index 96%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
index 58f4bee75..5006d3fbb 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
+package org.apache.unomi.shell.migration.actions;
/**
* Just a bean for a configuration property to be used during migration process
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
new file mode 100644
index 000000000..833099e4f
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration.actions;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.karaf.shell.api.console.Session;
+import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.unomi.shell.migration.actions.Migrate.MIGRATION_FS_ROOT_FOLDER;
+import static org.apache.unomi.shell.migration.actions.MigrationConfig.MIGRATION_HISTORY_RECOVER;
+
+/**
+ * This class allow for keeping track of the migration steps by persisting the steps and there state on the FileSystem,
+ * allowing for a migration to be able to restart from a failure in case it happens.
+ */
+public class MigrationHistory {
+
+ private static final Path MIGRATION_FS_HISTORY_FILE = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "history.json");
+
+ private enum MigrationStepState {
+ STARTED,
+ COMPLETED
+ }
+
+ public MigrationHistory(Session session, MigrationConfig migrationConfig) {
+ this.session = session;
+ this.migrationConfig = migrationConfig;
+ this.objectMapper = new ObjectMapper();
+
+ }
+
+ private final Session session;
+ private final MigrationConfig migrationConfig;
+ private final ObjectMapper objectMapper;
+
+ private Map<String, MigrationStepState> history = new HashMap<>();
+
+ /**
+ * Try to recover from a previous run
+ * I case we found an existing history we will ask if we want to recover or if we want to restart from the beginning
+ * (it is also configurable using the conf: recoverFromHistory)
+ */
+ protected void tryRecover() throws IOException {
+ if (Files.exists(MIGRATION_FS_HISTORY_FILE)) {
+ if (migrationConfig.getBoolean(MIGRATION_HISTORY_RECOVER, session)) {
+ history = objectMapper.readValue(MIGRATION_FS_HISTORY_FILE.toFile(), new TypeReference<Map<String, MigrationStepState>>() {});
+ } else {
+ clean();
+ }
+ }
+ }
+
+ /**
+ * this method allow for migration step execution:
+ * - in case the history already contains the given stepKey as COMPLETED, then the step won't be executed
+ * - in case the history doesn't contain the given stepKey, then the step will be executed
+ * Also this method is keeping track of the history by persisting it on the FileSystem.
+ *
+ * @param stepKey the key of the given step
+ * @param step the step to be performed
+ * @throws IOException
+ */
+ public void performMigrationStep(String stepKey, MigrationStep step) throws Exception {
+ if (step == null || stepKey == null) {
+ throw new IllegalArgumentException("Migration step and/or key cannot be null");
+ }
+
+ // check if step already exists in history:
+ MigrationStepState stepState = history.get(stepKey);
+ if (stepState != MigrationStepState.COMPLETED) {
+ updateStep(stepKey, MigrationStepState.STARTED);
+ step.execute();
+ updateStep(stepKey, MigrationStepState.COMPLETED);
+ } else {
+ ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " already completed in previous run");
+ }
+ }
+
+ /**
+ * Clean history from FileSystem
+ * @throws IOException
+ */
+ protected void clean() throws IOException {
+ Files.deleteIfExists(MIGRATION_FS_HISTORY_FILE);
+ }
+
+ private void updateStep(String stepKey, MigrationStepState stepState) throws IOException {
+ ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " reach: " + stepState);
+ history.put(stepKey, stepState);
+ objectMapper.writeValue(MIGRATION_FS_HISTORY_FILE.toFile(), history);
+ }
+
+ /**
+ * A simple migration step to be performed
+ */
+ public interface MigrationStep {
+ /**
+ * Do you migration a safe and unitary way, so that in case this step fail it can be re-executed safely
+ */
+ void execute() throws Exception;
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
similarity index 97%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
index 0355781bb..15533f056 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
+package org.apache.unomi.shell.migration.actions;
import groovy.lang.Script;
import org.apache.commons.io.IOUtils;
@@ -39,7 +39,7 @@ import java.util.regex.Pattern;
*/
public class MigrationScript implements Comparable<MigrationScript> {
- private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-([\\w|.]+).groovy$");
+ private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d+.\\d+.\\d+)-(\\d+)-([\\w|.]+).groovy$");
private final String script;
private Script compiledScript;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
index 072e97035..8c39fadea 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
@@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.MigrationUtils;
import org.json.JSONArray;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
index 5ad9c28ac..8fab2ee9d 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
@@ -19,15 +19,13 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpRequestException;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.osgi.framework.BundleContext;
-import org.osgi.service.component.annotations.Component;
import java.io.IOException;
-import java.util.Map;
public class MigrationTo122 implements Migration {
private CloseableHttpClient httpClient;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index 074a7fabf..c73bdab01 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -19,7 +19,7 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.json.JSONArray;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 6297a8951..9840b0184 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -23,6 +23,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
+import org.apache.unomi.shell.migration.actions.MigrationHistory;
import org.json.JSONArray;
import org.json.JSONObject;
import org.osgi.framework.Bundle;
@@ -141,7 +142,12 @@ public class MigrationUtils {
}
public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
- String newIndexSettings, String painlessScript) throws IOException {
+ String newIndexSettings, String painlessScript, MigrationHistory history) throws Exception {
+ if (indexName.endsWith("-cloned")) {
+ // We should never reIndex a clone ...
+ return;
+ }
+
String indexNameCloned = indexName + "-cloned";
String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json")
@@ -150,18 +156,34 @@ public class MigrationUtils {
String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json");
- // Set original index as readOnly
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName + "/_settings", setIndexReadOnlyRequest, null);
- // Clone the original index for backup
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_clone/" + indexNameCloned, null, null);
- // Delete original index
- HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
- // Recreate the original index with new mappings
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName, newIndexSettings, null);
- // Reindex data from clone
- HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
- // Remove clone
- HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
+ history.performMigrationStep("Reindex step for: " + indexName + " (clone creation)", () -> {
+ // Delete clone in case it already exists, could be incomplete from a previous reindex attempt, so better create a fresh one.
+ if (indexExists(httpClient, esAddress, indexNameCloned)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
+ }
+ // Set original index as readOnly
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName + "/_settings", setIndexReadOnlyRequest, null);
+ // Clone the original index for backup
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_clone/" + indexNameCloned, null, null);
+ });
+
+ history.performMigrationStep("Reindex step for: " + indexName + " (recreate the index and perform the re-indexation)", () -> {
+ // Delete original index if it still exists
+ if (indexExists(httpClient, esAddress, indexName)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
+ }
+ // Recreate the original index with new mappings
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName, newIndexSettings, null);
+ // Reindex data from clone
+ HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
+ });
+
+ history.performMigrationStep("Reindex step for: " + indexName + " (delete clone)", () -> {
+ // Delete original index if it still exists
+ if (indexExists(httpClient, esAddress, indexNameCloned)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
+ }
+ });
}
public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index 6ad9e9b8f..9dc340405 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -1,5 +1,8 @@
import groovy.json.JsonSlurper
+import org.apache.http.impl.client.CloseableHttpClient
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -22,49 +25,73 @@ import java.time.Instant
* limitations under the License.
*/
-Instant migrationTime = Instant.now();
+MigrationHistory history = migrationHistory
+CloseableHttpClient client = httpClient
+Instant migrationTime = Instant.now()
def jsonSlurper = new JsonSlurper()
-String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
String aliasIndex = indexPrefix + "-profilealias"
String profileIndex = indexPrefix + "-profile"
-// create alias index
-if (!MigrationUtils.indexExists(httpClient, esAddress, aliasIndex)) {
- String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, profileIndex, mapping)
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + aliasIndex, newIndexSettings, null)
- // scroll search on profiles merged
+history.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
+ if (!MigrationUtils.indexExists(client, esAddress, aliasIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, profileIndex, mapping)
+ HttpUtils.executePutRequest(client, esAddress + "/" + aliasIndex, newIndexSettings, null)
+ }
+})
+
+history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles", () -> {
+ String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
String profileMergedSearchRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_search.json")
- MigrationUtils.scrollQuery(httpClient, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", new MigrationUtils.ScrollCallback() {
- @Override
- void execute(String hits) {
- // create aliases for those merged profiles and delete them.
- def jsonHits = jsonSlurper.parseText(hits)
- ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " profile alias to create")
- final StringBuilder bulkSaveRequest = new StringBuilder()
- jsonHits.each {
- jsonHit -> {
- // check that master still exists before creating alias:
- def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + profileIndex + "/_doc/" + jsonHit._source.mergedWith, null))
- if (masterProfile.found) {
- bulkSaveRequest.append(aliasSaveBulkRequest
- .replace("##itemId##", jsonHit._source.itemId)
- .replace("##profileId##", jsonHit._source.mergedWith)
- .replace("##migrationTime##", migrationTime.toString()))
- }
+
+ MigrationUtils.scrollQuery(client, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", hits -> {
+ // create aliases for those merged profiles and delete them.
+ def jsonHits = jsonSlurper.parseText(hits)
+ ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " existing profiles merged")
+ final StringBuilder bulkSaveRequest = new StringBuilder()
+
+ jsonHits.each {
+ jsonHit -> {
+ // check that master still exists and that no alias exist for this profile yet
+ def mergedProfileId = jsonHit._source.itemId
+ def masterProfileId = jsonHit._source.mergedWith
+ def masterProfileExists = false
+ def aliasAlreadyExists = false
+
+ try {
+ def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + profileIndex + "/_doc/" + masterProfileId, null))
+ masterProfileExists = masterProfile.found
+ } catch (HttpRequestException e) {
+ // can happen in case response code > 400 due to item not exist in ElasticSearch
+ }
+
+ try {
+ def existingAlias = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + aliasIndex + "/_doc/" + mergedProfileId, null));
+ aliasAlreadyExists = existingAlias.found
+ } catch (HttpRequestException e) {
+ // can happen in case of response code > 400 due to item not exist in ElasticSearch
+ }
+
+ if (masterProfileExists && !aliasAlreadyExists) {
+ bulkSaveRequest.append(aliasSaveBulkRequest
+ .replace("##itemId##", mergedProfileId)
+ .replace("##profileId##", masterProfileId)
+ .replace("##migrationTime##", migrationTime.toString()))
}
}
- if (bulkSaveRequest.length() > 0) {
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
- }
+ }
+
+ if (bulkSaveRequest.length() > 0) {
+ HttpUtils.executePostRequest(client, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
})
+})
- // delete existing merged profiles
+history.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
-}
\ No newline at end of file
+ HttpUtils.executePostRequest(client, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
+})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 771c3ec70..273389cff 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -1,5 +1,8 @@
import groovy.json.JsonSlurper
+import org.apache.http.impl.client.CloseableHttpClient
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -20,6 +23,8 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
+CloseableHttpClient client = httpClient
def jsonSlurper = new JsonSlurper()
String searchScopesRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/scope_search.json")
String saveScopeRequestBulk = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scope_save_bulk.ndjson")
@@ -27,24 +32,47 @@ String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
String scopeIndex = indexPrefix + "-scope"
-// Create scope index:
-if (!MigrationUtils.indexExists(httpClient, esAddress, scopeIndex)) {
- String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, indexPrefix + "-profile", mapping)
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + scopeIndex, newIndexSettings, null)
+history.performMigrationStep("2.0.0-create-scope-index", () -> {
+ if (!MigrationUtils.indexExists(client, esAddress, scopeIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, indexPrefix + "-profile", mapping)
+ HttpUtils.executePutRequest(client, esAddress + "/" + scopeIndex, newIndexSettings, null)
+ }
+})
+history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
// search existing scopes from event
- def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
+ def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(client, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
ConsoleUtils.printMessage(session, "Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
// create scopes
def buckets = searchResponse.aggregations.scopes.buckets
if (buckets != null && buckets.size() > 0) {
final StringBuilder bulkSaveRequest = new StringBuilder()
+
buckets.each {
- bucket -> bulkSaveRequest.append(saveScopeRequestBulk.replace("##scope##", bucket.key))
+ bucket -> {
+ // Filter empty scope from existing events
+ if (bucket.key) {
+ // check that the scope doesn't already exists
+ def scopeAlreadyExists = false
+ try {
+ def existingScope = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + scopeIndex + "/_doc/" + bucket.key, null));
+ scopeAlreadyExists = existingScope.found
+ } catch (HttpRequestException e) {
+ // can happen in case response code > 400 due to item not exist in ElasticSearch
+ }
+
+ if (!scopeAlreadyExists) {
+ bulkSaveRequest.append(saveScopeRequestBulk.replace("##scope##", bucket.key))
+ }
+ }
+ }
+ }
+
+ if (bulkSaveRequest.length() > 0) {
+ HttpUtils.executePostRequest(client, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
-}
\ No newline at end of file
+})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 24603d75e..4d1df4791 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -1,3 +1,4 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -17,6 +18,7 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
@@ -27,5 +29,5 @@ indicesToReindex.each { indexToReindex ->
String realIndexName = "${indexPrefix}-${indexToReindex.toLowerCase()}"
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, mappingFileName)
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, realIndexName, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, realIndexName, newIndexSettings, null)
+ MigrationUtils.reIndex(httpClient, bundleContext, esAddress, realIndexName, newIndexSettings, null, history)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index 1fae7ffdb..b0bc12d6e 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -1,3 +1,4 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -17,6 +18,7 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
@@ -24,4 +26,4 @@ String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBod
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, indexPrefix + "-profile", mapping)
MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"))
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), history)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index aaad0599b..492c7ae1a 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -1,3 +1,4 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -18,12 +19,14 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
-// Remove all internal events that are no more persisted
-String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
-HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+history.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
+ String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
+ HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+})
// Reindex the rest of the events
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
@@ -32,5 +35,5 @@ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.
Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, "${indexPrefix}-event-")
eventIndices.each { eventIndex ->
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, eventIndex, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript)
+ MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, history)
}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index 1921ba5cf..b5dda03ac 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -17,8 +17,13 @@
# Migration config used for silent migration
+# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
# esAddress = http://localhost:9200
# esLogin = elastic
# esPassword = password
# httpClient.trustAllCertificates = true
-# indexPrefix = context
\ No newline at end of file
+# indexPrefix = context
+
+# Should the migration try to recover from a previous run ?
+# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
+# recoverFromHistory = true
\ No newline at end of file