You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/16 21:16:12 UTC
[unomi] branch master updated: UNOMI-627: migration recovery system in case of step failure (#473)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 45bab5233 UNOMI-627: migration recovery system in case of step failure (#473)
45bab5233 is described below
commit 45bab523346c9ef23c759676a5eb37f5aac83c47
Author: kevan Jahanshahi <ke...@jahia.com>
AuthorDate: Tue Aug 16 23:16:08 2022 +0200
UNOMI-627: migration recovery system in case of step failure (#473)
---
.../test/java/org/apache/unomi/itests/AllITs.java | 2 +
.../unomi/itests/migration/Migrate16xTo200IT.java | 7 +-
.../apache/unomi/itests/migration/MigrationIT.java | 72 ++++++++++++
.../migrate-11.0.0-01-failingMigration.groovy | 39 ++++---
.../migrate-11.0.0-01-successMigration.groovy | 38 ++++---
.../migration/org.apache.unomi.migration.cfg | 5 +
tools/shell-commands/pom.xml | 17 ++-
.../apache/unomi/shell/migration/Migration.java | 2 +-
.../unomi/shell/migration/actions/Migrate.java | 27 +++--
.../migration/{ => actions}/MigrationConfig.java | 4 +-
.../{ => actions}/MigrationConfigProperty.java | 2 +-
.../shell/migration/actions/MigrationHistory.java | 124 +++++++++++++++++++++
.../migration/{ => actions}/MigrationScript.java | 4 +-
.../unomi/shell/migration/impl/MigrationTo121.java | 2 +-
.../unomi/shell/migration/impl/MigrationTo122.java | 4 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 2 +-
.../shell/migration/utils/MigrationUtils.java | 48 +++++---
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 91 +++++++++------
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 48 ++++++--
.../migrate-2.0.0-05-globalReindex.groovy | 4 +-
.../migrate-2.0.0-10-profileReindex.groovy | 4 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 11 +-
.../main/resources/org.apache.unomi.migration.cfg | 7 +-
23 files changed, 444 insertions(+), 120 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index 1db16621a..a968eaa57 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -19,6 +19,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xTo200IT;
import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.MigrationIT;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
@@ -31,6 +32,7 @@ import org.junit.runners.Suite.SuiteClasses;
@RunWith(Suite.class)
@SuiteClasses({
Migrate16xTo200IT.class,
+ MigrationIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
ConditionESQueryBuilderIT.class,
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
index 921ae4ab1..115182ed8 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
@@ -55,7 +55,12 @@ public class Migrate16xTo200IT extends BaseIT {
}
// Do migrate the data set
- executeCommand("unomi:migrate 1.6.0 true");
+ String commandResults = executeCommand("unomi:migrate 1.6.0 true");
+
+ // Prin the resulted output in the karaf shell directly
+ System.out.println("Migration command output results:");
+ System.out.println(commandResults);
+
// Call super for starting Unomi and wait for the complete startup
super.waitForStartup();
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
new file mode 100644
index 000000000..94f546c01
--- /dev/null
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests.migration;
+
+import graphql.Assert;
+import org.apache.unomi.itests.BaseIT;
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class MigrationIT extends BaseIT {
+ protected static final Path BASE_DIRECTORIES = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts");
+ private static final String FAILING_SCRIPT_NAME = "migrate-11.0.0-01-failingMigration.groovy";
+ private static final String SUCCESS_SCRIPT_NAME = "migrate-11.0.0-01-successMigration.groovy";
+ private static final String FAILING_SCRIPT_RESOURCE = "migration/" + FAILING_SCRIPT_NAME;
+ private static final String SUCCESS_SCRIPT_RESOURCE = "migration/" + SUCCESS_SCRIPT_NAME;
+ protected static final Path FAILING_SCRIPT_FS_PATH = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts", FAILING_SCRIPT_NAME);
+ protected static final Path SUCCESS_SCRIPT_FS_PATH = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts", SUCCESS_SCRIPT_NAME);
+
+ @Test
+ public void checkMigrationRecoverySystem() throws Exception {
+ try {
+ Files.createDirectories(BASE_DIRECTORIES);
+
+ Files.write(FAILING_SCRIPT_FS_PATH, bundleResourceAsString(FAILING_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
+ String failingResult = executeCommand("unomi:migrate 10.0.0 true");
+ System.out.println("Intentional failing migration result:");
+ System.out.println(failingResult);
+ // step 4 and 5 should not be contains, step 3 is failing
+ // Only step 1, 2 and 3 should be performed.
+ Assert.assertTrue(failingResult.contains("inside step 1"));
+ Assert.assertTrue(failingResult.contains("inside step 2"));
+ Assert.assertTrue(failingResult.contains("inside step 3"));
+ Assert.assertTrue(!failingResult.contains("inside step 4"));
+ Assert.assertTrue(!failingResult.contains("inside step 5"));
+ Files.deleteIfExists(FAILING_SCRIPT_FS_PATH);
+
+ Files.write(SUCCESS_SCRIPT_FS_PATH, bundleResourceAsString(SUCCESS_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
+ String successResult = executeCommand("unomi:migrate 10.0.0 true");
+ System.out.println("Success recovered from failing migration result:");
+ System.out.println(successResult);
+ // step 1 and 2 should not be contains, they passed on first attempt.
+ // Only step 3, 4 and 5 should be performed.
+ Assert.assertTrue(!successResult.contains("inside step 1"));
+ Assert.assertTrue(!successResult.contains("inside step 2"));
+ Assert.assertTrue(successResult.contains("inside step 3"));
+ Assert.assertTrue(successResult.contains("inside step 4"));
+ Assert.assertTrue(successResult.contains("inside step 5"));
+ Files.deleteIfExists(SUCCESS_SCRIPT_FS_PATH);
+ } finally {
+ Files.deleteIfExists(FAILING_SCRIPT_FS_PATH);
+ Files.deleteIfExists(SUCCESS_SCRIPT_FS_PATH);
+ }
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
similarity index 51%
copy from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
copy to itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
index 58f4bee75..84ea5e908 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
@@ -1,3 +1,6 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -14,25 +17,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
-/**
- * Just a bean for a configuration property to be used during migration process
- */
-public class MigrationConfigProperty {
- String description;
- String defaultValue;
+MigrationHistory history = migrationHistory
+history.performMigrationStep("step 1", () -> {
+ ConsoleUtils.printMessage(session, "inside step 1")
+})
+
+history.performMigrationStep("step 2", () -> {
+ ConsoleUtils.printMessage(session, "inside step 2")
+})
- public MigrationConfigProperty(String description, String defaultValue) {
- this.description = description;
- this.defaultValue = defaultValue;
- }
+history.performMigrationStep("step 3", () -> {
+ ConsoleUtils.printMessage(session, "inside step 3")
+ throw new RuntimeException("Intentional failure !")
+})
- public String getDescription() {
- return description;
- }
+history.performMigrationStep("step 4", () -> {
+ ConsoleUtils.printMessage(session, "inside step 4")
+})
- public String getDefaultValue() {
- return defaultValue;
- }
-}
+history.performMigrationStep("step 5", () -> {
+ ConsoleUtils.printMessage(session, "inside step 5")
+})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
similarity index 53%
copy from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
copy to itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
index 58f4bee75..44c4403ad 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
@@ -1,3 +1,6 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -14,25 +17,24 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
-/**
- * Just a bean for a configuration property to be used during migration process
- */
-public class MigrationConfigProperty {
- String description;
- String defaultValue;
+MigrationHistory history = migrationHistory
+history.performMigrationStep("step 1", () -> {
+ ConsoleUtils.printMessage(session, "inside step 1")
+})
+
+history.performMigrationStep("step 2", () -> {
+ ConsoleUtils.printMessage(session, "inside step 2")
+})
- public MigrationConfigProperty(String description, String defaultValue) {
- this.description = description;
- this.defaultValue = defaultValue;
- }
+history.performMigrationStep("step 3", () -> {
+ ConsoleUtils.printMessage(session, "inside step 3")
+})
- public String getDescription() {
- return description;
- }
+history.performMigrationStep("step 4", () -> {
+ ConsoleUtils.printMessage(session, "inside step 4")
+})
- public String getDefaultValue() {
- return defaultValue;
- }
-}
+history.performMigrationStep("step 5", () -> {
+ ConsoleUtils.printMessage(session, "inside step 5")
+})
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg b/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
index 26cfca708..d96120c28 100644
--- a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
+++ b/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
@@ -17,7 +17,12 @@
# Migration config used for silent migration
+# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
esAddress = http://localhost:9400
esLogin =
httpClient.trustAllCertificates = true
indexPrefix = context
+
+# Should the migration try to recover from a previous run ?
+# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
+recoverFromHistory = true
\ No newline at end of file
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 1b73ab209..0a3df585f 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -93,6 +93,20 @@
<version>${groovy.version}</version>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>${version.jackson.core}</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>${version.jackson.databind}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
@@ -103,7 +117,8 @@
<configuration>
<instructions>
<Export-Package>
- org.apache.unomi.shell.migration.utils
+ org.apache.unomi.shell.migration.utils,
+ org.apache.unomi.shell.migration.actions,
</Export-Package>
<Embed-Dependency>*;scope=compile|runtime</Embed-Dependency>
<DynamicImport-Package>*</DynamicImport-Package>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
index c7d9ece8c..0113b3854 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
@@ -18,10 +18,10 @@ package org.apache.unomi.shell.migration;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.osgi.framework.BundleContext;
import java.io.IOException;
-import java.util.Map;
/**
* @author dgaillard
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index ea5afdfde..a21b5d8c3 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -31,8 +31,6 @@ import org.apache.karaf.shell.api.action.Command;
import org.apache.karaf.shell.api.action.lifecycle.Reference;
import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.MigrationConfig;
-import org.apache.unomi.shell.migration.MigrationScript;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.osgi.framework.*;
@@ -47,13 +45,16 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.unomi.shell.migration.MigrationConfig.*;
+import static org.apache.unomi.shell.migration.actions.MigrationConfig.*;
@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " +
"It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process.")
@Service
public class Migrate implements Action {
+ protected static final String MIGRATION_FS_ROOT_FOLDER = "migration";
+ protected static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
+
@Reference
Session session;
@@ -101,6 +102,9 @@ public class Migrate implements Action {
// reset migration config from previous stored users choices.
migrationConfig.reset();
+ Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
+ MigrationHistory migrationHistory = new MigrationHistory(session, migrationConfig);
+ migrationHistory.tryRecover();
// Handle credentials
CredentialsProvider credentialsProvider = null;
@@ -115,7 +119,7 @@ public class Migrate implements Action {
try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session), credentialsProvider)) {
// Compile scripts
- scripts = parseScripts(scripts, session, httpClient, migrationConfig);
+ scripts = parseScripts(scripts, session, httpClient, migrationConfig, migrationHistory);
// Start migration
ConsoleUtils.printMessage(session, "Starting migration process from version: " + originVersion);
@@ -130,6 +134,9 @@ public class Migrate implements Action {
ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
}
+
+ // We clean history, migration is successful
+ migrationHistory.clean();
}
return null;
@@ -152,7 +159,7 @@ public class Migrate implements Action {
.collect(Collectors.toCollection(TreeSet::new));
}
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig) {
+ private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
return scripts.stream()
@@ -160,7 +167,7 @@ public class Migrate implements Action {
// fallback on current bundle if the scripts is not provided by OSGI
Bundle scriptBundle = migrateScript.getBundle() != null ? migrateScript.getBundle() : bundleContext.getBundle();
if (!shellsPerBundle.containsKey(scriptBundle.getSymbolicName())) {
- shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig));
+ shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig, migrationHistory));
}
migrateScript.setCompiledScript(shellsPerBundle.get(scriptBundle.getSymbolicName()).parse(migrateScript.getScript()));
})
@@ -186,13 +193,12 @@ public class Migrate implements Action {
private Set<MigrationScript> loadFileSystemScripts() throws IOException {
// check migration folder exists
- Path migrationFolder = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts");
- if (!Files.isDirectory(migrationFolder)) {
+ if (!Files.isDirectory(MIGRATION_FS_SCRIPTS_FOLDER)) {
return Collections.emptySet();
}
List<Path> paths;
- try (Stream<Path> walk = Files.walk(migrationFolder)) {
+ try (Stream<Path> walk = Files.walk(MIGRATION_FS_SCRIPTS_FOLDER)) {
paths = walk
.filter(path -> !Files.isDirectory(path))
.filter(path -> path.toString().toLowerCase().endsWith("groovy"))
@@ -206,13 +212,14 @@ public class Migrate implements Action {
return migrationScripts;
}
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig) {
+ private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
groovyShell.setVariable("session", session);
groovyShell.setVariable("httpClient", httpClient);
groovyShell.setVariable("migrationConfig", migrationConfig);
+ groovyShell.setVariable("migrationHistory", migrationHistory);
groovyShell.setVariable("bundleContext", bundle.getBundleContext());
return groovyShell;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
similarity index 94%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
index 41f37aac7..df01f0de0 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
+package org.apache.unomi.shell.migration.actions;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
@@ -46,6 +46,7 @@ public class MigrationConfig {
public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
public static final String TOTAL_FIELDS_LIMIT = "mapping.total_fields.limit";
public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
+ public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
private static final Map<String, MigrationConfigProperty> configProperties;
static {
@@ -59,6 +60,7 @@ public class MigrationConfig {
m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
m.put(TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
m.put(MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
+ m.put(MIGRATION_HISTORY_RECOVER, new MigrationConfigProperty("We found an existing migration attempt, should we restart from it ? (this will avoid redoing steps already completed successfully) (yes/no)", null));
configProperties = Collections.unmodifiableMap(m);
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
similarity index 96%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
index 58f4bee75..5006d3fbb 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
+package org.apache.unomi.shell.migration.actions;
/**
* Just a bean for a configuration property to be used during migration process
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
new file mode 100644
index 000000000..833099e4f
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration.actions;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.karaf.shell.api.console.Session;
+import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.unomi.shell.migration.actions.Migrate.MIGRATION_FS_ROOT_FOLDER;
+import static org.apache.unomi.shell.migration.actions.MigrationConfig.MIGRATION_HISTORY_RECOVER;
+
+/**
+ * This class allow for keeping track of the migration steps by persisting the steps and there state on the FileSystem,
+ * allowing for a migration to be able to restart from a failure in case it happens.
+ */
+public class MigrationHistory {
+
+ private static final Path MIGRATION_FS_HISTORY_FILE = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "history.json");
+
+ private enum MigrationStepState {
+ STARTED,
+ COMPLETED
+ }
+
+ public MigrationHistory(Session session, MigrationConfig migrationConfig) {
+ this.session = session;
+ this.migrationConfig = migrationConfig;
+ this.objectMapper = new ObjectMapper();
+
+ }
+
+ private final Session session;
+ private final MigrationConfig migrationConfig;
+ private final ObjectMapper objectMapper;
+
+ private Map<String, MigrationStepState> history = new HashMap<>();
+
+ /**
+ * Try to recover from a previous run
+ * I case we found an existing history we will ask if we want to recover or if we want to restart from the beginning
+ * (it is also configurable using the conf: recoverFromHistory)
+ */
+ protected void tryRecover() throws IOException {
+ if (Files.exists(MIGRATION_FS_HISTORY_FILE)) {
+ if (migrationConfig.getBoolean(MIGRATION_HISTORY_RECOVER, session)) {
+ history = objectMapper.readValue(MIGRATION_FS_HISTORY_FILE.toFile(), new TypeReference<Map<String, MigrationStepState>>() {});
+ } else {
+ clean();
+ }
+ }
+ }
+
+ /**
+ * this method allow for migration step execution:
+ * - in case the history already contains the given stepKey as COMPLETED, then the step won't be executed
+ * - in case the history doesn't contain the given stepKey, then the step will be executed
+ * Also this method is keeping track of the history by persisting it on the FileSystem.
+ *
+ * @param stepKey the key of the given step
+ * @param step the step to be performed
+ * @throws IOException
+ */
+ public void performMigrationStep(String stepKey, MigrationStep step) throws Exception {
+ if (step == null || stepKey == null) {
+ throw new IllegalArgumentException("Migration step and/or key cannot be null");
+ }
+
+ // check if step already exists in history:
+ MigrationStepState stepState = history.get(stepKey);
+ if (stepState != MigrationStepState.COMPLETED) {
+ updateStep(stepKey, MigrationStepState.STARTED);
+ step.execute();
+ updateStep(stepKey, MigrationStepState.COMPLETED);
+ } else {
+ ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " already completed in previous run");
+ }
+ }
+
+ /**
+ * Clean history from FileSystem
+ * @throws IOException
+ */
+ protected void clean() throws IOException {
+ Files.deleteIfExists(MIGRATION_FS_HISTORY_FILE);
+ }
+
+ private void updateStep(String stepKey, MigrationStepState stepState) throws IOException {
+ ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " reach: " + stepState);
+ history.put(stepKey, stepState);
+ objectMapper.writeValue(MIGRATION_FS_HISTORY_FILE.toFile(), history);
+ }
+
+ /**
+ * A simple migration step to be performed
+ */
+ public interface MigrationStep {
+ /**
+ * Do you migration a safe and unitary way, so that in case this step fail it can be re-executed safely
+ */
+ void execute() throws Exception;
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
similarity index 97%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
index 0355781bb..15533f056 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration;
+package org.apache.unomi.shell.migration.actions;
import groovy.lang.Script;
import org.apache.commons.io.IOUtils;
@@ -39,7 +39,7 @@ import java.util.regex.Pattern;
*/
public class MigrationScript implements Comparable<MigrationScript> {
- private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-([\\w|.]+).groovy$");
+ private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d+.\\d+.\\d+)-(\\d+)-([\\w|.]+).groovy$");
private final String script;
private Script compiledScript;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
index 072e97035..8c39fadea 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
@@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.MigrationUtils;
import org.json.JSONArray;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
index 5ad9c28ac..8fab2ee9d 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
@@ -19,15 +19,13 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpRequestException;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.osgi.framework.BundleContext;
-import org.osgi.service.component.annotations.Component;
import java.io.IOException;
-import java.util.Map;
public class MigrationTo122 implements Migration {
private CloseableHttpClient httpClient;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index 074a7fabf..c73bdab01 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -19,7 +19,7 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
+import org.apache.unomi.shell.migration.actions.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.json.JSONArray;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 6297a8951..9840b0184 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -23,6 +23,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
+import org.apache.unomi.shell.migration.actions.MigrationHistory;
import org.json.JSONArray;
import org.json.JSONObject;
import org.osgi.framework.Bundle;
@@ -141,7 +142,12 @@ public class MigrationUtils {
}
public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
- String newIndexSettings, String painlessScript) throws IOException {
+ String newIndexSettings, String painlessScript, MigrationHistory history) throws Exception {
+ if (indexName.endsWith("-cloned")) {
+ // We should never reIndex a clone ...
+ return;
+ }
+
String indexNameCloned = indexName + "-cloned";
String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json")
@@ -150,18 +156,34 @@ public class MigrationUtils {
String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json");
- // Set original index as readOnly
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName + "/_settings", setIndexReadOnlyRequest, null);
- // Clone the original index for backup
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_clone/" + indexNameCloned, null, null);
- // Delete original index
- HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
- // Recreate the original index with new mappings
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName, newIndexSettings, null);
- // Reindex data from clone
- HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
- // Remove clone
- HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
+ history.performMigrationStep("Reindex step for: " + indexName + " (clone creation)", () -> {
+ // Delete clone in case it already exists, could be incomplete from a previous reindex attempt, so better create a fresh one.
+ if (indexExists(httpClient, esAddress, indexNameCloned)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
+ }
+ // Set original index as readOnly
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName + "/_settings", setIndexReadOnlyRequest, null);
+ // Clone the original index for backup
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_clone/" + indexNameCloned, null, null);
+ });
+
+ history.performMigrationStep("Reindex step for: " + indexName + " (recreate the index and perform the re-indexation)", () -> {
+ // Delete original index if it still exists
+ if (indexExists(httpClient, esAddress, indexName)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
+ }
+ // Recreate the original index with new mappings
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + indexName, newIndexSettings, null);
+ // Reindex data from clone
+ HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
+ });
+
+ history.performMigrationStep("Reindex step for: " + indexName + " (delete clone)", () -> {
+ // Delete original index if it still exists
+ if (indexExists(httpClient, esAddress, indexNameCloned)) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
+ }
+ });
}
public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index 6ad9e9b8f..9dc340405 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -1,5 +1,8 @@
import groovy.json.JsonSlurper
+import org.apache.http.impl.client.CloseableHttpClient
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -22,49 +25,73 @@ import java.time.Instant
* limitations under the License.
*/
-Instant migrationTime = Instant.now();
+MigrationHistory history = migrationHistory
+CloseableHttpClient client = httpClient
+Instant migrationTime = Instant.now()
def jsonSlurper = new JsonSlurper()
-String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
String aliasIndex = indexPrefix + "-profilealias"
String profileIndex = indexPrefix + "-profile"
-// create alias index
-if (!MigrationUtils.indexExists(httpClient, esAddress, aliasIndex)) {
- String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, profileIndex, mapping)
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + aliasIndex, newIndexSettings, null)
- // scroll search on profiles merged
+history.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
+ if (!MigrationUtils.indexExists(client, esAddress, aliasIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, profileIndex, mapping)
+ HttpUtils.executePutRequest(client, esAddress + "/" + aliasIndex, newIndexSettings, null)
+ }
+})
+
+history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles", () -> {
+ String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
String profileMergedSearchRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_search.json")
- MigrationUtils.scrollQuery(httpClient, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", new MigrationUtils.ScrollCallback() {
- @Override
- void execute(String hits) {
- // create aliases for those merged profiles and delete them.
- def jsonHits = jsonSlurper.parseText(hits)
- ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " profile alias to create")
- final StringBuilder bulkSaveRequest = new StringBuilder()
- jsonHits.each {
- jsonHit -> {
- // check that master still exists before creating alias:
- def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + profileIndex + "/_doc/" + jsonHit._source.mergedWith, null))
- if (masterProfile.found) {
- bulkSaveRequest.append(aliasSaveBulkRequest
- .replace("##itemId##", jsonHit._source.itemId)
- .replace("##profileId##", jsonHit._source.mergedWith)
- .replace("##migrationTime##", migrationTime.toString()))
- }
+
+ MigrationUtils.scrollQuery(client, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", hits -> {
+ // create aliases for those merged profiles and delete them.
+ def jsonHits = jsonSlurper.parseText(hits)
+ ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " existing profiles merged")
+ final StringBuilder bulkSaveRequest = new StringBuilder()
+
+ jsonHits.each {
+ jsonHit -> {
+ // check that master still exists and that no alias exist for this profile yet
+ def mergedProfileId = jsonHit._source.itemId
+ def masterProfileId = jsonHit._source.mergedWith
+ def masterProfileExists = false
+ def aliasAlreadyExists = false
+
+ try {
+ def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + profileIndex + "/_doc/" + masterProfileId, null))
+ masterProfileExists = masterProfile.found
+ } catch (HttpRequestException e) {
+ // can happen in case response code > 400 due to item not exist in ElasticSearch
+ }
+
+ try {
+ def existingAlias = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + aliasIndex + "/_doc/" + mergedProfileId, null));
+ aliasAlreadyExists = existingAlias.found
+ } catch (HttpRequestException e) {
+ // can happen in case of response code > 400 due to item not exist in ElasticSearch
+ }
+
+ if (masterProfileExists && !aliasAlreadyExists) {
+ bulkSaveRequest.append(aliasSaveBulkRequest
+ .replace("##itemId##", mergedProfileId)
+ .replace("##profileId##", masterProfileId)
+ .replace("##migrationTime##", migrationTime.toString()))
}
}
- if (bulkSaveRequest.length() > 0) {
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
- }
+ }
+
+ if (bulkSaveRequest.length() > 0) {
+ HttpUtils.executePostRequest(client, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
})
+})
- // delete existing merged profiles
+history.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
-}
\ No newline at end of file
+ HttpUtils.executePostRequest(client, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
+})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 771c3ec70..273389cff 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -1,5 +1,8 @@
import groovy.json.JsonSlurper
+import org.apache.http.impl.client.CloseableHttpClient
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -20,6 +23,8 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
+CloseableHttpClient client = httpClient
def jsonSlurper = new JsonSlurper()
String searchScopesRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/scope_search.json")
String saveScopeRequestBulk = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scope_save_bulk.ndjson")
@@ -27,24 +32,47 @@ String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
String scopeIndex = indexPrefix + "-scope"
-// Create scope index:
-if (!MigrationUtils.indexExists(httpClient, esAddress, scopeIndex)) {
- String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, indexPrefix + "-profile", mapping)
- HttpUtils.executePutRequest(httpClient, esAddress + "/" + scopeIndex, newIndexSettings, null)
+history.performMigrationStep("2.0.0-create-scope-index", () -> {
+ if (!MigrationUtils.indexExists(client, esAddress, scopeIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, indexPrefix + "-profile", mapping)
+ HttpUtils.executePutRequest(client, esAddress + "/" + scopeIndex, newIndexSettings, null)
+ }
+})
+history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
// search existing scopes from event
- def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
+ def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(client, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
ConsoleUtils.printMessage(session, "Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
// create scopes
def buckets = searchResponse.aggregations.scopes.buckets
if (buckets != null && buckets.size() > 0) {
final StringBuilder bulkSaveRequest = new StringBuilder()
+
buckets.each {
- bucket -> bulkSaveRequest.append(saveScopeRequestBulk.replace("##scope##", bucket.key))
+ bucket -> {
+ // Filter empty scope from existing events
+ if (bucket.key) {
+ // check that the scope doesn't already exists
+ def scopeAlreadyExists = false
+ try {
+ def existingScope = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + scopeIndex + "/_doc/" + bucket.key, null));
+ scopeAlreadyExists = existingScope.found
+ } catch (HttpRequestException e) {
+ // can happen in case response code > 400 due to item not exist in ElasticSearch
+ }
+
+ if (!scopeAlreadyExists) {
+ bulkSaveRequest.append(saveScopeRequestBulk.replace("##scope##", bucket.key))
+ }
+ }
+ }
+ }
+
+ if (bulkSaveRequest.length() > 0) {
+ HttpUtils.executePostRequest(client, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
-}
\ No newline at end of file
+})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 24603d75e..4d1df4791 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -1,3 +1,4 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -17,6 +18,7 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
@@ -27,5 +29,5 @@ indicesToReindex.each { indexToReindex ->
String realIndexName = "${indexPrefix}-${indexToReindex.toLowerCase()}"
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, mappingFileName)
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, realIndexName, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, realIndexName, newIndexSettings, null)
+ MigrationUtils.reIndex(httpClient, bundleContext, esAddress, realIndexName, newIndexSettings, null, history)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index 1fae7ffdb..b0bc12d6e 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -1,3 +1,4 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -17,6 +18,7 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
@@ -24,4 +26,4 @@ String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBod
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, indexPrefix + "-profile", mapping)
MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"))
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), history)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index aaad0599b..492c7ae1a 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -1,3 +1,4 @@
+import org.apache.unomi.shell.migration.actions.MigrationHistory
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -18,12 +19,14 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
+MigrationHistory history = migrationHistory
String esAddress = migrationConfig.getString("esAddress", session)
String indexPrefix = migrationConfig.getString("indexPrefix", session)
-// Remove all internal events that are no more persisted
-String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
-HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+history.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
+ String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
+ HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+})
// Reindex the rest of the events
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
@@ -32,5 +35,5 @@ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.
Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, "${indexPrefix}-event-")
eventIndices.each { eventIndex ->
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, eventIndex, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript)
+ MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, history)
}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index 1921ba5cf..b5dda03ac 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -17,8 +17,13 @@
# Migration config used for silent migration
+# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
# esAddress = http://localhost:9200
# esLogin = elastic
# esPassword = password
# httpClient.trustAllCertificates = true
-# indexPrefix = context
\ No newline at end of file
+# indexPrefix = context
+
+# Should the migration try to recover from a previous run ?
+# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
+# recoverFromHistory = true
\ No newline at end of file