You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/18 08:56:54 UTC
[unomi] branch master updated: UNOMI-622: provide automation support for migration (#474)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 3cad03ec6 UNOMI-622: provide automation support for migration (#474)
3cad03ec6 is described below
commit 3cad03ec6f09272043a660797ee5c398660dfde8
Author: kevan Jahanshahi <ke...@jahia.com>
AuthorDate: Thu Aug 18 10:56:49 2022 +0200
UNOMI-622: provide automation support for migration (#474)
* UNOMI-622: provide automation support for migration
* UNOMI-622: keep migration history on file system after successful migration
* UNOMI-622: add final status for migration finished
* UNOMI-622: improve reindex by using other index settings for recovery system to work in case reindexation failure
* UNOMI-622: improve reindex by using other index settings for recovery system to work in case reindexation failure
* UNOMI-622: stabilize test try
---
.../test/java/org/apache/unomi/itests/BaseIT.java | 1 -
.../java/org/apache/unomi/itests/SegmentIT.java | 7 +
.../apache/unomi/itests/migration/MigrationIT.java | 24 +-
.../migrate-11.0.0-01-failingMigration.groovy | 25 +-
.../migrate-11.0.0-01-successMigration.groovy | 25 +-
.../migration/org.apache.unomi.migration.cfg | 28 --
.../main/resources/etc/custom.system.properties | 5 +
tools/shell-commands/pom.xml | 2 +-
.../apache/unomi/shell/migration/Migration.java | 11 +-
.../unomi/shell/migration/MigrationService.java | 38 +++
.../unomi/shell/migration/actions/Migrate.java | 187 +-------------
.../shell/migration/actions/MigrationHistory.java | 124 ---------
.../unomi/shell/migration/impl/MigrationTo121.java | 21 +-
.../unomi/shell/migration/impl/MigrationTo122.java | 23 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 52 ++--
.../{actions => service}/MigrationConfig.java | 58 +----
.../MigrationConfigProperty.java | 4 +-
.../shell/migration/service/MigrationContext.java | 284 +++++++++++++++++++++
.../{actions => service}/MigrationScript.java | 18 +-
.../MigrationServiceImpl.java} | 141 +++++-----
.../unomi/shell/migration/utils/ConsoleUtils.java | 101 --------
.../shell/migration/utils/MigrationUtils.java | 10 +-
.../internal/UnomiManagementServiceImpl.java | 67 ++++-
.../migration/migrate-1.2.1-00-migrateTags.groovy | 2 +-
.../migrate-1.2.2-00-deleteOldIndexTemplate.groovy | 2 +-
.../migrate-1.5.0-00-elasticSearch7.4.groovy | 2 +-
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 35 ++-
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 29 +--
.../migrate-2.0.0-05-globalReindex.groovy | 16 +-
.../migrate-2.0.0-10-profileReindex.groovy | 14 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 20 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 62 -----
.../main/resources/org.apache.unomi.migration.cfg | 13 +-
33 files changed, 645 insertions(+), 806 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
index dc103951d..f1bb826b7 100644
--- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
@@ -235,7 +235,6 @@ public abstract class BaseIT extends KarafTestSupport {
System.out.println("==== Configuring container");
Option[] options = new Option[]{
replaceConfigurationFile("etc/org.apache.unomi.router.cfg", new File("src/test/resources/org.apache.unomi.router.cfg")),
- replaceConfigurationFile("etc/org.apache.unomi.migration.cfg", new File("src/test/resources/migration/org.apache.unomi.migration.cfg")),
replaceConfigurationFile("data/tmp/1-basic-test.csv", new File("src/test/resources/1-basic-test.csv")),
replaceConfigurationFile("data/tmp/recurrent_import/2-surfers-test.csv", new File("src/test/resources/2-surfers-test.csv")),
diff --git a/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java
index b1a3c0626..4dcefebdc 100644
--- a/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java
@@ -255,6 +255,13 @@ public class SegmentIT extends BaseIT {
Event testEvent = new Event("negative-test-event-type", null, profile, null, null, profile,
Date.from(localDate.atStartOfDay(defaultZoneId).toInstant()));
testEvent.setPersistent(true);
+
+ // wait for segment auto generated rule to be available
+ keepTrying("The segment auto generated rule should be available to handle the test event",
+ () -> rulesService.getMatchingRules(testEvent),
+ rules -> rules.size() > 0, 1000, 20);
+
+ // send the event
int changes = eventService.send(testEvent);
if ((changes & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) {
profileService.save(profile);
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
index 94f546c01..a7fa068d1 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
@@ -16,8 +16,8 @@
*/
package org.apache.unomi.itests.migration;
-import graphql.Assert;
import org.apache.unomi.itests.BaseIT;
+import org.junit.Assert;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
@@ -25,6 +25,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import static org.junit.Assert.fail;
+
public class MigrationIT extends BaseIT {
protected static final Path BASE_DIRECTORIES = Paths.get(System.getProperty( "karaf.data" ), "migration", "scripts");
private static final String FAILING_SCRIPT_NAME = "migrate-11.0.0-01-failingMigration.groovy";
@@ -40,16 +42,12 @@ public class MigrationIT extends BaseIT {
Files.createDirectories(BASE_DIRECTORIES);
Files.write(FAILING_SCRIPT_FS_PATH, bundleResourceAsString(FAILING_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
- String failingResult = executeCommand("unomi:migrate 10.0.0 true");
- System.out.println("Intentional failing migration result:");
- System.out.println(failingResult);
- // step 4 and 5 should not be contains, step 3 is failing
- // Only step 1, 2 and 3 should be performed.
- Assert.assertTrue(failingResult.contains("inside step 1"));
- Assert.assertTrue(failingResult.contains("inside step 2"));
- Assert.assertTrue(failingResult.contains("inside step 3"));
- Assert.assertTrue(!failingResult.contains("inside step 4"));
- Assert.assertTrue(!failingResult.contains("inside step 5"));
+ try {
+ executeCommand("unomi:migrate 10.0.0 true");
+ fail("Migration should have failed and crashed by Exception throwing");
+ } catch (Exception e) {
+ // this is expected, the script fail at step 3
+ }
Files.deleteIfExists(FAILING_SCRIPT_FS_PATH);
Files.write(SUCCESS_SCRIPT_FS_PATH, bundleResourceAsString(SUCCESS_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
@@ -58,8 +56,8 @@ public class MigrationIT extends BaseIT {
System.out.println(successResult);
// step 1 and 2 should not be contains, they passed on first attempt.
// Only step 3, 4 and 5 should be performed.
- Assert.assertTrue(!successResult.contains("inside step 1"));
- Assert.assertTrue(!successResult.contains("inside step 2"));
+ Assert.assertFalse(successResult.contains("inside step 1"));
+ Assert.assertFalse(successResult.contains("inside step 2"));
Assert.assertTrue(successResult.contains("inside step 3"));
Assert.assertTrue(successResult.contains("inside step 4"));
Assert.assertTrue(successResult.contains("inside step 5"));
diff --git a/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
index 84ea5e908..3e801d725 100644
--- a/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
@@ -1,5 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,24 +17,24 @@ import org.apache.unomi.shell.migration.utils.ConsoleUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-history.performMigrationStep("step 1", () -> {
- ConsoleUtils.printMessage(session, "inside step 1")
+MigrationContext context = migrationContext
+context.performMigrationStep("step 1", () -> {
+ context.printMessage("inside step 1")
})
-history.performMigrationStep("step 2", () -> {
- ConsoleUtils.printMessage(session, "inside step 2")
+context.performMigrationStep("step 2", () -> {
+ context.printMessage("inside step 2")
})
-history.performMigrationStep("step 3", () -> {
- ConsoleUtils.printMessage(session, "inside step 3")
+context.performMigrationStep("step 3", () -> {
+ context.printMessage("inside step 3")
throw new RuntimeException("Intentional failure !")
})
-history.performMigrationStep("step 4", () -> {
- ConsoleUtils.printMessage(session, "inside step 4")
+context.performMigrationStep("step 4", () -> {
+ context.printMessage("inside step 4")
})
-history.performMigrationStep("step 5", () -> {
- ConsoleUtils.printMessage(session, "inside step 5")
+context.performMigrationStep("step 5", () -> {
+ context.printMessage("inside step 5")
})
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
index 44c4403ad..93db092b9 100644
--- a/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
@@ -1,5 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,23 +17,23 @@ import org.apache.unomi.shell.migration.utils.ConsoleUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-history.performMigrationStep("step 1", () -> {
- ConsoleUtils.printMessage(session, "inside step 1")
+MigrationContext context = migrationContext
+context.performMigrationStep("step 1", () -> {
+ context.printMessage("inside step 1")
})
-history.performMigrationStep("step 2", () -> {
- ConsoleUtils.printMessage(session, "inside step 2")
+context.performMigrationStep("step 2", () -> {
+ context.printMessage("inside step 2")
})
-history.performMigrationStep("step 3", () -> {
- ConsoleUtils.printMessage(session, "inside step 3")
+context.performMigrationStep("step 3", () -> {
+ context.printMessage("inside step 3")
})
-history.performMigrationStep("step 4", () -> {
- ConsoleUtils.printMessage(session, "inside step 4")
+context.performMigrationStep("step 4", () -> {
+ context.printMessage("inside step 4")
})
-history.performMigrationStep("step 5", () -> {
- ConsoleUtils.printMessage(session, "inside step 5")
+context.performMigrationStep("step 5", () -> {
+ context.printMessage("inside step 5")
})
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg b/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
deleted file mode 100644
index d96120c28..000000000
--- a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Migration config used for silent migration
-
-# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
-esAddress = http://localhost:9400
-esLogin =
-httpClient.trustAllCertificates = true
-indexPrefix = context
-
-# Should the migration try to recover from a previous run ?
-# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
-recoverFromHistory = true
\ No newline at end of file
diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties
index fe09b36b9..d0525ca5d 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -419,3 +419,8 @@ org.apache.unomi.weatherUpdate.url.attributes=${env:UNOMI_WEATHERUPDATE_URL_ATTR
## Settings for GraphQL ##
#######################################################################################################################
org.apache.unomi.graphql.feature.activated=${env:UNOMI_GRAPHQL_FEATURE_ACTIVATED:-false}
+
+#######################################################################################################################
+## Settings for migration ##
+#######################################################################################################################
+org.apache.unomi.migration.recoverFromHistory=${env:UNOMI_MIGRATION_RECOVER_FROM_HISTORY:-true}
\ No newline at end of file
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 0a3df585f..df8b398c9 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -118,7 +118,7 @@
<instructions>
<Export-Package>
org.apache.unomi.shell.migration.utils,
- org.apache.unomi.shell.migration.actions,
+ org.apache.unomi.shell.migration.service,
</Export-Package>
<Embed-Dependency>*;scope=compile|runtime</Embed-Dependency>
<DynamicImport-Package>*</DynamicImport-Package>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
index 0113b3854..b3e103b0b 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
@@ -18,7 +18,8 @@ package org.apache.unomi.shell.migration;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.osgi.framework.BundleContext;
import java.io.IOException;
@@ -30,12 +31,10 @@ import java.io.IOException;
public interface Migration {
/**
* This method is called to execute the migration
- * @param session the shell's session
- * @param httpClient CloseableHttpClient
- * @param migrationConfig config used to perform the migration, like esAddress, trustAllCertificates, etc ...
+ * @param migrationContext the current migration context (config, history, messaging, logging)
* @param bundleContext the bundle context object
* @throws IOException if there was an error while executing the migration
* @deprecated do groovy script for implementing new migrations
*/
- void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException;
-}
+ void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException;
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationService.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationService.java
new file mode 100644
index 000000000..7671771b6
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationService.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration;
+
+import org.apache.karaf.shell.api.console.Session;
+
+/**
+ * Simple OSGI service used for migrating Unomi data
+ */
+public interface MigrationService {
+
+ /**
+ * This will Migrate your data in ES to be compliant with current Unomi version.
+ * It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg,
+ * if no configuration is provided then questions will be prompted during the migration process.
+ * (only in case you are in karaf shell context, if not, a missing configuration will fail the migration process)
+ *
+ * @param originVersion Origin version without suffix/qualifier (e.g: 1.2.0)
+ * @param skipConfirmation Should the confirmation before starting the migration process be skipped ? (only supported in karaf shell context)
+ * @param session Karaf shell session, for execution in Karaf shell context, null otherwise
+ * @throws Exception
+ */
+ void migrateUnomi(String originVersion, boolean skipConfirmation, Session session) throws Exception;
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index a21b5d8c3..1b5a683e9 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -16,53 +16,25 @@
*/
package org.apache.unomi.shell.migration.actions;
-import groovy.lang.GroovyClassLoader;
-import groovy.lang.GroovyShell;
-import groovy.util.GroovyScriptEngine;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.action.Action;
import org.apache.karaf.shell.api.action.Argument;
import org.apache.karaf.shell.api.action.Command;
import org.apache.karaf.shell.api.action.lifecycle.Reference;
import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
-import org.apache.unomi.shell.migration.utils.HttpUtils;
-import org.osgi.framework.*;
-import org.osgi.framework.wiring.BundleWiring;
-
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.unomi.shell.migration.actions.MigrationConfig.*;
+import org.apache.unomi.shell.migration.MigrationService;
@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " +
- "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process.")
+ "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, " +
+ "if no configuration is provided then questions will be prompted during the migration process.")
@Service
public class Migrate implements Action {
- protected static final String MIGRATION_FS_ROOT_FOLDER = "migration";
- protected static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
-
@Reference
Session session;
@Reference
- BundleContext bundleContext;
-
- @Reference
- MigrationConfig migrationConfig;
+ MigrationService migrationService;
@Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
private String originVersion;
@@ -71,156 +43,7 @@ public class Migrate implements Action {
private boolean skipConfirmation = false;
public Object execute() throws Exception {
- // Load migration scrips
- Set<MigrationScript> scripts = loadOSGIScripts();
- scripts.addAll(loadFileSystemScripts());
-
- if (originVersion == null) {
- displayMigrations(scripts);
- ConsoleUtils.printMessage(session, "Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
- return null;
- }
-
- // Check that there is some migration scripts available from given version
- Version fromVersion = new Version(originVersion);
- scripts = filterScriptsFromVersion(scripts, fromVersion);
- if (scripts.size() == 0) {
- ConsoleUtils.printMessage(session, "No migration scripts available found starting from version: " + originVersion);
- return null;
- } else {
- ConsoleUtils.printMessage(session, "The following migration scripts starting from version: " + originVersion + " will be executed.");
- displayMigrations(scripts);
- }
-
- // Check for user approval before migrate
- if (!skipConfirmation && ConsoleUtils.askUserWithAuthorizedAnswer(session,
- "[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ",
- Arrays.asList("yes", "no")).equalsIgnoreCase("no")) {
- ConsoleUtils.printMessage(session, "Migration process aborted");
- return null;
- }
-
- // reset migration config from previous stored users choices.
- migrationConfig.reset();
- Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
- MigrationHistory migrationHistory = new MigrationHistory(session, migrationConfig);
- migrationHistory.tryRecover();
-
- // Handle credentials
- CredentialsProvider credentialsProvider = null;
- String login = migrationConfig.getString(CONFIG_ES_LOGIN, session);
- if (StringUtils.isNotEmpty(login)) {
- credentialsProvider = new BasicCredentialsProvider();
- UsernamePasswordCredentials credentials
- = new UsernamePasswordCredentials(login, migrationConfig.getString(CONFIG_ES_PASSWORD, session));
- credentialsProvider.setCredentials(AuthScope.ANY, credentials);
- }
-
- try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session), credentialsProvider)) {
-
- // Compile scripts
- scripts = parseScripts(scripts, session, httpClient, migrationConfig, migrationHistory);
-
- // Start migration
- ConsoleUtils.printMessage(session, "Starting migration process from version: " + originVersion);
- for (MigrationScript migrateScript : scripts) {
- ConsoleUtils.printMessage(session, "Starting execution of: " + migrateScript);
- try {
- migrateScript.getCompiledScript().run();
- } catch (Exception e) {
- ConsoleUtils.printException(session, "Error executing: " + migrateScript, e);
- return null;
- }
-
- ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
- }
-
- // We clean history, migration is successful
- migrationHistory.clean();
- }
-
+ migrationService.migrateUnomi(originVersion, skipConfirmation, session);
return null;
}
-
- private void displayMigrations(Set<MigrationScript> scripts) {
- Version previousVersion = new Version("0.0.0");
- for (MigrationScript migration : scripts) {
- if (migration.getVersion().getMajor() > previousVersion.getMajor() || migration.getVersion().getMinor() > previousVersion.getMinor()) {
- ConsoleUtils.printMessage(session, "From " + migration.getVersion().getMajor() + "." + migration.getVersion().getMinor() + ".0:");
- }
- ConsoleUtils.printMessage(session, "- " + migration);
- previousVersion = migration.getVersion();
- }
- }
-
- private Set<MigrationScript> filterScriptsFromVersion(Set<MigrationScript> scripts, Version fromVersion) {
- return scripts.stream()
- .filter(migrateScript -> fromVersion.compareTo(migrateScript.getVersion()) < 0)
- .collect(Collectors.toCollection(TreeSet::new));
- }
-
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
- Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
-
- return scripts.stream()
- .peek(migrateScript -> {
- // fallback on current bundle if the scripts is not provided by OSGI
- Bundle scriptBundle = migrateScript.getBundle() != null ? migrateScript.getBundle() : bundleContext.getBundle();
- if (!shellsPerBundle.containsKey(scriptBundle.getSymbolicName())) {
- shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig, migrationHistory));
- }
- migrateScript.setCompiledScript(shellsPerBundle.get(scriptBundle.getSymbolicName()).parse(migrateScript.getScript()));
- })
- .collect(Collectors.toCollection(TreeSet::new));
- }
-
- private Set<MigrationScript> loadOSGIScripts() throws IOException {
- SortedSet<MigrationScript> migrationScripts = new TreeSet<>();
- for (Bundle bundle : bundleContext.getBundles()) {
- Enumeration<URL> scripts = bundle.findEntries("META-INF/cxs/migration", "*.groovy", true);
- if (scripts != null) {
- // check for shell
-
- while (scripts.hasMoreElements()) {
- URL scriptURL = scripts.nextElement();
- migrationScripts.add(new MigrationScript(scriptURL, bundle));
- }
- }
- }
-
- return migrationScripts;
- }
-
- private Set<MigrationScript> loadFileSystemScripts() throws IOException {
- // check migration folder exists
- if (!Files.isDirectory(MIGRATION_FS_SCRIPTS_FOLDER)) {
- return Collections.emptySet();
- }
-
- List<Path> paths;
- try (Stream<Path> walk = Files.walk(MIGRATION_FS_SCRIPTS_FOLDER)) {
- paths = walk
- .filter(path -> !Files.isDirectory(path))
- .filter(path -> path.toString().toLowerCase().endsWith("groovy"))
- .collect(Collectors.toList());
- }
-
- SortedSet<MigrationScript> migrationScripts = new TreeSet<>();
- for (Path path : paths) {
- migrationScripts.add(new MigrationScript(path.toUri().toURL(), null));
- }
- return migrationScripts;
- }
-
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
- GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
- GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
- GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
- groovyShell.setVariable("session", session);
- groovyShell.setVariable("httpClient", httpClient);
- groovyShell.setVariable("migrationConfig", migrationConfig);
- groovyShell.setVariable("migrationHistory", migrationHistory);
- groovyShell.setVariable("bundleContext", bundle.getBundleContext());
- return groovyShell;
- }
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
deleted file mode 100644
index 833099e4f..000000000
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.shell.migration.actions;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.unomi.shell.migration.actions.Migrate.MIGRATION_FS_ROOT_FOLDER;
-import static org.apache.unomi.shell.migration.actions.MigrationConfig.MIGRATION_HISTORY_RECOVER;
-
-/**
- * This class allow for keeping track of the migration steps by persisting the steps and there state on the FileSystem,
- * allowing for a migration to be able to restart from a failure in case it happens.
- */
-public class MigrationHistory {
-
- private static final Path MIGRATION_FS_HISTORY_FILE = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "history.json");
-
- private enum MigrationStepState {
- STARTED,
- COMPLETED
- }
-
- public MigrationHistory(Session session, MigrationConfig migrationConfig) {
- this.session = session;
- this.migrationConfig = migrationConfig;
- this.objectMapper = new ObjectMapper();
-
- }
-
- private final Session session;
- private final MigrationConfig migrationConfig;
- private final ObjectMapper objectMapper;
-
- private Map<String, MigrationStepState> history = new HashMap<>();
-
- /**
- * Try to recover from a previous run
- * I case we found an existing history we will ask if we want to recover or if we want to restart from the beginning
- * (it is also configurable using the conf: recoverFromHistory)
- */
- protected void tryRecover() throws IOException {
- if (Files.exists(MIGRATION_FS_HISTORY_FILE)) {
- if (migrationConfig.getBoolean(MIGRATION_HISTORY_RECOVER, session)) {
- history = objectMapper.readValue(MIGRATION_FS_HISTORY_FILE.toFile(), new TypeReference<Map<String, MigrationStepState>>() {});
- } else {
- clean();
- }
- }
- }
-
- /**
- * this method allow for migration step execution:
- * - in case the history already contains the given stepKey as COMPLETED, then the step won't be executed
- * - in case the history doesn't contain the given stepKey, then the step will be executed
- * Also this method is keeping track of the history by persisting it on the FileSystem.
- *
- * @param stepKey the key of the given step
- * @param step the step to be performed
- * @throws IOException
- */
- public void performMigrationStep(String stepKey, MigrationStep step) throws Exception {
- if (step == null || stepKey == null) {
- throw new IllegalArgumentException("Migration step and/or key cannot be null");
- }
-
- // check if step already exists in history:
- MigrationStepState stepState = history.get(stepKey);
- if (stepState != MigrationStepState.COMPLETED) {
- updateStep(stepKey, MigrationStepState.STARTED);
- step.execute();
- updateStep(stepKey, MigrationStepState.COMPLETED);
- } else {
- ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " already completed in previous run");
- }
- }
-
- /**
- * Clean history from FileSystem
- * @throws IOException
- */
- protected void clean() throws IOException {
- Files.deleteIfExists(MIGRATION_FS_HISTORY_FILE);
- }
-
- private void updateStep(String stepKey, MigrationStepState stepState) throws IOException {
- ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " reach: " + stepState);
- history.put(stepKey, stepState);
- objectMapper.writeValue(MIGRATION_FS_HISTORY_FILE.toFile(), history);
- }
-
- /**
- * A simple migration step to be performed
- */
- public interface MigrationStep {
- /**
- * Do you migration a safe and unitary way, so that in case this step fail it can be re-executed safely
- */
- void execute() throws Exception;
- }
-}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
index 8c39fadea..f19b55970 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
@@ -18,10 +18,9 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.apache.unomi.shell.migration.utils.MigrationUtils;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -36,23 +35,21 @@ import java.util.*;
public class MigrationTo121 implements Migration {
private CloseableHttpClient httpClient;
- private Session session;
private String esAddress;
private LinkedHashMap<String, List<String>> tagsStructurePriorTo130;
private List propsTaggedAsPersonalIdentifier = Arrays.asList("firstName", "lastName", "email", "phoneNumber", "address", "facebookId", "googleId", "linkedInId", "twitterId");
@Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- this.httpClient = httpClient;
- this.session = session;
- this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- migrateTags();
+ public void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException {
+ this.httpClient = migrationContext.getHttpClient();
+ this.esAddress = migrationContext.getConfigString(MigrationConfig.CONFIG_ES_ADDRESS);
+ migrateTags(migrationContext);
}
- private void migrateTags() throws IOException {
+ private void migrateTags(MigrationContext migrationContext) throws IOException {
initTagsStructurePriorTo130();
- String tagsOperation = ConsoleUtils.askUserWithAuthorizedAnswer(session, "How to manage tags?\n1. copy: will duplicate tags in systemTags property\n2. move: will move tags in systemTags property\n[1 - 2]: ", Arrays.asList("1", "2"));
- String removeNamespaceOnSystemTags = ConsoleUtils.askUserWithAuthorizedAnswer(session, "As we will copy/move the tags, do you wish to remove existing namespace on tags before copy/move in systemTags? (e.g: hidden.) (yes/no): ", Arrays.asList("yes", "no"));
+ String tagsOperation = migrationContext.askUserWithAuthorizedAnswer("How to manage tags?\n1. copy: will duplicate tags in systemTags property\n2. move: will move tags in systemTags property\n[1 - 2]: ", Arrays.asList("1", "2"));
+ String removeNamespaceOnSystemTags = migrationContext.askUserWithAuthorizedAnswer("As we will copy/move the tags, do you wish to remove existing namespace on tags before copy/move in systemTags? (e.g: hidden.) (yes/no): ", Arrays.asList("yes", "no"));
List<String> typeToMigrate = Arrays.asList("actionType", "conditionType", "campaign", "goal", "rule", "scoring", "segment", "userList");
for (String type : typeToMigrate) {
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
index 8fab2ee9d..3fab4c182 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
@@ -17,10 +17,9 @@
package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.apache.unomi.shell.migration.utils.HttpRequestException;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.osgi.framework.BundleContext;
@@ -29,30 +28,26 @@ import java.io.IOException;
public class MigrationTo122 implements Migration {
private CloseableHttpClient httpClient;
- private Session session;
private String esAddress;
@Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- this.httpClient = httpClient;
- this.session = session;
- this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- deleteOldIndexTemplate();
-
+ public void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException {
+ this.httpClient = migrationContext.getHttpClient();
+ this.esAddress = migrationContext.getConfigString(MigrationConfig.CONFIG_ES_ADDRESS);
+ deleteOldIndexTemplate(migrationContext);
}
- private void deleteOldIndexTemplate() throws IOException {
+ private void deleteOldIndexTemplate(MigrationContext migrationContext) throws IOException {
String oldMonthlyIndexTemplate = "context_monthlyindex";
try {
- ConsoleUtils.printMessage(session,"Deleting old monthly index template " + oldMonthlyIndexTemplate);
+ migrationContext.printMessage("Deleting old monthly index template " + oldMonthlyIndexTemplate);
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/_template/" + oldMonthlyIndexTemplate, null);
} catch (HttpRequestException e) {
if (e.getCode() == 404) {
- ConsoleUtils.printMessage(session,"Old monthly index template not found, skipping deletion");
+ migrationContext.printMessage("Old monthly index template not found, skipping deletion");
} else {
throw e;
}
}
-
}
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index c73bdab01..9131ea354 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -17,10 +17,9 @@
package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -38,13 +37,14 @@ public class MigrationTo150 implements Migration {
public static final String INDEX_DATE_PREFIX = "date-";
@Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- String esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- String es5Address = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
- String sourceIndexPrefix = migrationConfig.getString(MigrationConfig.INDEX_PREFIX, session);
- String destIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "TARGET index prefix (default: context) : ", "context");
- int numberOfShards = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of shards for TARGET (default: 5) : ", "5"));
- int numberOfReplicas = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of replicas for TARGET (default: 1) : ", "1"));
+ public void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException {
+ CloseableHttpClient httpClient = migrationContext.getHttpClient();
+ String esAddress = migrationContext.getConfigString(MigrationConfig.CONFIG_ES_ADDRESS);
+ String es5Address = migrationContext.askUserWithDefaultAnswer("SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
+ String sourceIndexPrefix = migrationContext.getConfigString(MigrationConfig.INDEX_PREFIX);
+ String destIndexPrefix = migrationContext.askUserWithDefaultAnswer("TARGET index prefix (default: context) : ", "context");
+ int numberOfShards = Integer.parseInt(migrationContext.askUserWithDefaultAnswer("Number of shards for TARGET (default: 5) : ", "5"));
+ int numberOfReplicas = Integer.parseInt(migrationContext.askUserWithDefaultAnswer("Number of replicas for TARGET (default: 1) : ", "1"));
Set<String> monthlyIndexTypes = new HashSet<>();
monthlyIndexTypes.add("event");
monthlyIndexTypes.add("session");
@@ -75,27 +75,27 @@ public class MigrationTo150 implements Migration {
if (!monthlyIndexTypes.contains(itemType)) {
String indexName = "geonameEntry".equals(itemType) ? "geonames" : sourceIndexPrefix;
- JSONObject es5TypeMapping = getES5TypeMapping(session, httpClient, es5Address, indexName, itemType);
+ JSONObject es5TypeMapping = getES5TypeMapping(httpClient, es5Address, indexName, itemType);
int es5MappingsTotalFieldsLimit = getES5MappingsTotalFieldsLimit(httpClient, es5Address, indexName);
String destIndexName = itemType.toLowerCase();
if (!indexExists(httpClient, esAddress, destIndexPrefix, destIndexName)) {
createESIndex(httpClient, esAddress, destIndexPrefix, destIndexName, numberOfShards, numberOfReplicas, es5MappingsTotalFieldsLimit, getMergedTypeMapping(es5TypeMapping, newTypeMapping));
- reIndex(session, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
+ reIndex(migrationContext, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
} else {
- ConsoleUtils.printMessage(session, "Index " + getIndexName(destIndexPrefix, itemType.toLowerCase()) + " already exists, skipping re-indexation...");
+ migrationContext.printMessage("Index " + getIndexName(destIndexPrefix, itemType.toLowerCase()) + " already exists, skipping re-indexation...");
}
} else {
for (String indexName : monthlyIndexNames) {
// we need to extract the date part
String datePart = indexName.substring(sourceIndexPrefix.length() + 1);
String destIndexName = itemType.toLowerCase() + "-" + INDEX_DATE_PREFIX + datePart;
- JSONObject es5TypeMapping = getES5TypeMapping(session, httpClient, es5Address, indexName, itemType);
+ JSONObject es5TypeMapping = getES5TypeMapping(httpClient, es5Address, indexName, itemType);
int es5MappingsTotalFieldsLimit = getES5MappingsTotalFieldsLimit(httpClient, es5Address, indexName);
if (!indexExists(httpClient, esAddress, destIndexPrefix, destIndexName)) {
createESIndex(httpClient, esAddress, destIndexPrefix, destIndexName, numberOfShards, numberOfReplicas, es5MappingsTotalFieldsLimit, getMergedTypeMapping(es5TypeMapping, newTypeMapping));
- reIndex(session, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
+ reIndex(migrationContext, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
} else {
- ConsoleUtils.printMessage(session, "Index " + getIndexName(destIndexPrefix, destIndexName) + " already exists, skipping re-indexation...");
+ migrationContext.printMessage("Index " + getIndexName(destIndexPrefix, destIndexName) + " already exists, skipping re-indexation...");
}
}
}
@@ -103,7 +103,7 @@ public class MigrationTo150 implements Migration {
}
long totalMigrationTime = System.currentTimeMillis() - startTime;
- ConsoleUtils.printMessage(session, "Migration operations completed in " + totalMigrationTime + "ms");
+ migrationContext.printMessage("Migration operations completed in " + totalMigrationTime + "ms");
}
private String loadMappingFile(URL predefinedMappingURL) throws IOException {
@@ -158,7 +158,7 @@ public class MigrationTo150 implements Migration {
HttpUtils.executePutRequest(httpClient, esAddress + "/" + getIndexName(indexPrefix, indexName), indexBody.toString(), null);
}
- private void reIndex(Session session, CloseableHttpClient httpClient,
+ private void reIndex(MigrationContext migrationContext, CloseableHttpClient httpClient,
String esAddress,
String es5Address,
String sourceIndexName,
@@ -176,31 +176,31 @@ public class MigrationTo150 implements Migration {
.put("dest", new JSONObject()
.put("index", destIndexName)
);
- ConsoleUtils.printMessage(session, "Reindexing " + sourceIndexName + " to " + destIndexName + "...");
+ migrationContext.printMessage("Reindexing " + sourceIndexName + " to " + destIndexName + "...");
long startTime = System.currentTimeMillis();
try {
String response = HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reindexSettings.toString(), null);
long reindexationTime = System.currentTimeMillis() - startTime;
- ConsoleUtils.printMessage(session, "Reindexing completed in " + reindexationTime + "ms. Result=" + response);
+ migrationContext.printMessage("Reindexing completed in " + reindexationTime + "ms. Result=" + response);
} catch (IOException ioe) {
- ConsoleUtils.printException(session, "Error executing reindexing", ioe);
- ConsoleUtils.printMessage(session, "Attempting to delete index " + destIndexName + " so that we can restart from this point...");
- deleteIndex(session, httpClient, esAddress, destIndexName);
+ migrationContext.printException("Error executing reindexing", ioe);
+ migrationContext.printMessage("Attempting to delete index " + destIndexName + " so that we can restart from this point...");
+ deleteIndex(migrationContext, httpClient, esAddress, destIndexName);
throw ioe;
}
}
- private void deleteIndex(Session session, CloseableHttpClient httpClient,
+ private void deleteIndex(MigrationContext migrationContext, CloseableHttpClient httpClient,
String esAddress,
String indexName) {
try {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
} catch (IOException ioe) {
- ConsoleUtils.printException(session, "Error attempting to delete index" + indexName, ioe);
+ migrationContext.printException("Error attempting to delete index" + indexName, ioe);
}
}
- private JSONObject getES5TypeMapping(Session session, CloseableHttpClient httpClient, String es5Address, String indexName, String typeName) throws IOException {
+ private JSONObject getES5TypeMapping(CloseableHttpClient httpClient, String es5Address, String indexName, String typeName) throws IOException {
String response = HttpUtils.executeGetRequest(httpClient, es5Address + "/" + indexName, null);
if (response != null) {
JSONObject indexInfo = new JSONObject(response).getJSONObject(indexName);
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
similarity index 60%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
index df01f0de0..33f7f321b 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
@@ -14,30 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
-import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Modified;
-import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
- * Service uses to aggregate different configuration needed by the migrations
- * Source of config:
- * - file system in OSGI config file: org.apache.unomi.migration.cfg
- * - user interactions in the console during the migration process
+ * Service uses to provide configuration information for the migration
*/
@Component(immediate = true, service = MigrationConfig.class, configurationPid = {"org.apache.unomi.migration"})
public class MigrationConfig {
public static final String CONFIG_ES_ADDRESS = "esAddress";
+ public static final String CONFIG_ES_ADDRESSES = "esAddresses";
+ public static final String CONFIG_ES_SSL_ENABLED = "esSSLEnabled";
public static final String CONFIG_ES_LOGIN = "esLogin";
public static final String CONFIG_ES_PASSWORD = "esPassword";
public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
@@ -48,10 +43,11 @@ public class MigrationConfig {
public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
- private static final Map<String, MigrationConfigProperty> configProperties;
+ protected static final Map<String, MigrationConfigProperty> configProperties;
static {
Map<String, MigrationConfigProperty> m = new HashMap<>();
- m.put(CONFIG_ES_ADDRESS, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: http://localhost:9200): ", "http://localhost:9200"));
+ m.put(CONFIG_ES_ADDRESSES, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: localhost:9200): ", "localhost:9200"));
+ m.put(CONFIG_ES_SSL_ENABLED, new MigrationConfigProperty("Should the ElasticSearch TARGET connection be established using SSL (https) protocol ? (yes/no)", null));
m.put(CONFIG_ES_LOGIN, new MigrationConfigProperty("Enter ElasticSearch TARGET login (default: none): ", ""));
m.put(CONFIG_ES_PASSWORD, new MigrationConfigProperty("Enter ElasticSearch TARGET password (default: none): ", ""));
m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ? (yes/no)", null));
@@ -64,47 +60,15 @@ public class MigrationConfig {
configProperties = Collections.unmodifiableMap(m);
}
- Map<String, String> initialConfig = new HashMap<>();
- Map<String, String> computeConfig = new HashMap<>();
+ private Map<String, String> config = new HashMap<>();
@Activate
@Modified
public void modified(Map<String, String> config) {
- initialConfig = config;
- reset();
+ this.config = config;
}
- /**
- * Used reset user choices to initial file system config (useful at the beginning of each new migrate session)
- */
- public void reset() {
- computeConfig.clear();
- computeConfig.putAll(initialConfig);
- }
-
- public String getString(String name, Session session) throws IOException {
- if (computeConfig.containsKey(name)) {
- return computeConfig.get(name);
- }
- if (configProperties.containsKey(name)) {
- MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
- String answer = ConsoleUtils.askUserWithDefaultAnswer(session, migrateConfigProperty.getDescription(), migrateConfigProperty.getDefaultValue());
- computeConfig.put(name, answer);
- return answer;
- }
- return null;
- }
-
- public boolean getBoolean(String name, Session session) throws IOException {
- if (computeConfig.containsKey(name)) {
- return Boolean.parseBoolean(computeConfig.get(name));
- }
- if (configProperties.containsKey(name)) {
- MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
- boolean answer = ConsoleUtils.askUserWithAuthorizedAnswer(session, migrateConfigProperty.getDescription(), Arrays.asList("yes", "no")).equalsIgnoreCase("yes");
- computeConfig.put(name, answer ? "true" : "false");
- return answer;
- }
- return false;
+ protected Map<String, String> getConfig() {
+ return this.config;
}
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfigProperty.java
similarity index 90%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfigProperty.java
index 5006d3fbb..37de28b8c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfigProperty.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
/**
* Just a bean for a configuration property to be used during migration process
@@ -23,7 +23,7 @@ public class MigrationConfigProperty {
String description;
String defaultValue;
- public MigrationConfigProperty(String description, String defaultValue) {
+ protected MigrationConfigProperty(String description, String defaultValue) {
this.description = description;
this.defaultValue = defaultValue;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
new file mode 100644
index 000000000..2f20353c2
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration.service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.karaf.shell.api.console.Session;
+import org.jline.reader.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
+import static org.apache.unomi.shell.migration.service.MigrationServiceImpl.MIGRATION_FS_ROOT_FOLDER;
+
+/**
+ * This class is instantiated for each migration process, it contains useful methods to handle the current migration lifecycle.
+ *
+ * This class allow for keeping track of the migration steps by persisting the steps and there state on the FileSystem,
+ * allowing for a migration to be able to restart from a failure in case it happens.
+ *
+ * This class allow also for logging the migration informations depending on the current context:
+ * - if executed in karaf shell using a karaf shell session, then the logging will be done in the shell console
+ * - if executed outside karaf shell using OSGI service direct, then the logging will be done using classical logger systems
+ *
+ * This class allow also to do a best effort on missing configuration information, by prompting questions in the karaf shell
+ * (not supported in case direct OSGI service usage)
+ */
+public class MigrationContext {
+ private static final Logger logger = LoggerFactory.getLogger(MigrationContext.class);
+
+ private static final Path MIGRATION_FS_HISTORY_FILE = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "history.json");
+
+ private enum MigrationStepState {
+ STARTED,
+ COMPLETED
+ }
+
+ protected MigrationContext(Session session, MigrationConfig migrationConfig) {
+ this.session = session;
+ this.migrationConfig = migrationConfig;
+ this.objectMapper = new ObjectMapper();
+ }
+
+ private final Session session;
+ private final MigrationConfig migrationConfig;
+ private final ObjectMapper objectMapper;
+ private CloseableHttpClient httpClient;
+
+ private Map<String, MigrationStepState> history = new HashMap<>();
+ private Map<String, String> userConfig = new HashMap<>();
+
+ /**
+ * Try to recover from a previous run
+ * I case we found an existing history we will ask if we want to recover or if we want to restart from the beginning
+ * (it is also configurable using the conf: recoverFromHistory)
+ */
+ protected void tryRecoverFromHistory() throws IOException {
+ if (Files.exists(MIGRATION_FS_HISTORY_FILE)) {
+ if (getConfigBoolean(MIGRATION_HISTORY_RECOVER)) {
+ history = objectMapper.readValue(MIGRATION_FS_HISTORY_FILE.toFile(), new TypeReference<Map<String, MigrationStepState>>() {});
+ } else {
+ cleanHistory();
+ }
+ }
+ }
+
+ /**
+ * this method allow for migration step execution:
+ * - in case the history already contains the given stepKey as COMPLETED, then the step won't be executed
+ * - in case the history doesn't contain the given stepKey, then the step will be executed
+ * Also this method is keeping track of the history by persisting it on the FileSystem.
+ *
+ * @param stepKey the key of the given step
+ * @param step the step to be performed
+ * @throws IOException
+ */
+ public void performMigrationStep(String stepKey, MigrationStep step) throws Exception {
+ if (step == null || stepKey == null) {
+ throw new IllegalArgumentException("Migration step and/or key cannot be null");
+ }
+
+ // check if step already exists in history:
+ MigrationStepState stepState = history.get(stepKey);
+ if (stepState != MigrationStepState.COMPLETED) {
+ updateHistoryStep(stepKey, MigrationStepState.STARTED);
+ step.execute();
+ updateHistoryStep(stepKey, MigrationStepState.COMPLETED);
+ } else {
+ printMessage("Migration step: " + stepKey + " already completed in previous run");
+ }
+ }
+
+ /**
+ * Clean history from FileSystem
+ * @throws IOException
+ */
+ protected void cleanHistory() throws IOException {
+ Files.deleteIfExists(MIGRATION_FS_HISTORY_FILE);
+ }
+
+ /**
+ * This will ask a question to the user and return the default answer if the user does not answer.
+ *
+ * @param msg String message to ask
+ * @param defaultAnswer String default answer
+ * @return the user's answer
+ * @throws IOException if there was a problem reading input from the console
+ */
+ public String askUserWithDefaultAnswer(String msg, String defaultAnswer) throws IOException {
+ String answer = promptMessageToUser(msg);
+ if (StringUtils.isBlank(answer)) {
+ return defaultAnswer;
+ }
+ return answer;
+ }
+
+ /**
+ * This method allow you to ask a question to the user.
+ * The answer is controlled before being return so the question will be ask until the user enter one the authorized answer
+ *
+ * @param msg String message to ask
+ * @param authorizedAnswer Array of possible answer, all answer must be in lower case
+ * @return the user answer
+ * @throws IOException if there was an error retrieving an answer from the user on the console
+ */
+ public String askUserWithAuthorizedAnswer(String msg, List<String> authorizedAnswer) throws IOException {
+ String answer;
+ do {
+ answer = promptMessageToUser(msg);
+ } while (!authorizedAnswer.contains(answer.toLowerCase()));
+ return answer;
+ }
+
+ /**
+ * This method allow you to prompt a message to the user.
+ * No control is done on the answer provided by the user.
+ *
+ * @param msg String message to prompt
+ * @return the user answer
+ */
+ public String promptMessageToUser(String msg) {
+ if (session == null) {
+ throw new IllegalStateException("Cannot prompt message: " + msg + " to user. " +
+ "(In case you are using the migration tool out of Karaf shell context, please check the migration configuration: org.apache.unomi.migration.cfg)");
+ }
+ LineReader reader = (LineReader) session.get(".jline.reader");
+ return reader.readLine(msg, null);
+ }
+
+ /**
+ * Print a message in the console.
+ * @param msg the message to print out with a newline
+ */
+ public void printMessage(String msg) {
+ if (session == null) {
+ logger.info(msg);
+ } else {
+ PrintStream writer = session.getConsole();
+ writer.println(msg);
+ }
+ }
+
+ /**
+ * Print an exception along with a message in the console.
+ * @param msg the message to print out with a newline
+ * @param t the exception to dump in the shell console after the message
+ */
+ public void printException(String msg, Throwable t) {
+ if (session == null) {
+ logger.error(msg, t);
+ } else {
+ PrintStream writer = session.getConsole();
+ writer.println(msg);
+ t.printStackTrace(writer);
+ }
+ }
+
+ /**
+ * Get config for property name, in case the property doesn't exist on file system config file
+ * Best effort will be made to prompt question in karaf shell to get the needed information
+ *
+ * @param name the name of the property
+ * @return the value of the property
+ * @throws IOException
+ */
+ public String getConfigString(String name) throws IOException {
+ // special handling for esAddress that need to be built
+ if (CONFIG_ES_ADDRESS.equals(name)) {
+ String esAddresses = getConfigString(CONFIG_ES_ADDRESSES);
+ boolean sslEnabled = getConfigBoolean(CONFIG_ES_SSL_ENABLED);
+ return (sslEnabled ? "https://" : "http://") + esAddresses.split(",")[0].trim();
+ }
+
+ if (migrationConfig.getConfig().containsKey(name)) {
+ return migrationConfig.getConfig().get(name);
+ }
+ if (userConfig.containsKey(name)) {
+ return userConfig.get(name);
+ }
+ if (configProperties.containsKey(name)) {
+ MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
+ String answer = askUserWithDefaultAnswer(migrateConfigProperty.getDescription(), migrateConfigProperty.getDefaultValue());
+ userConfig.put(name, answer);
+ return answer;
+ }
+ return null;
+ }
+
+ /**
+ * Get config for property name, in case the property doesn't exist on file system config file
+ * Best effort will be made to prompt question in karaf shell to get the needed information
+ *
+ * @param name the name of the property
+ * @return the value of the property
+ * @throws IOException
+ */
+ public boolean getConfigBoolean(String name) throws IOException {
+ if (migrationConfig.getConfig().containsKey(name)) {
+ return Boolean.parseBoolean(migrationConfig.getConfig().get(name));
+ }
+ if (userConfig.containsKey(name)) {
+ return Boolean.parseBoolean(userConfig.get(name));
+ }
+ if (configProperties.containsKey(name)) {
+ MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
+ boolean answer = askUserWithAuthorizedAnswer(migrateConfigProperty.getDescription(), Arrays.asList("yes", "no")).equalsIgnoreCase("yes");
+ userConfig.put(name, answer ? "true" : "false");
+ return answer;
+ }
+ return false;
+ }
+
+ /**
+ * This HTTP client is configured to be used for ElasticSearch requests to be able to perform migrations requests.
+ * @return the http client.
+ */
+ public CloseableHttpClient getHttpClient() {
+ return httpClient;
+ }
+
+
+ private void updateHistoryStep(String stepKey, MigrationStepState stepState) throws IOException {
+ printMessage("Migration step: " + stepKey + " reach: " + stepState);
+ history.put(stepKey, stepState);
+ objectMapper.writeValue(MIGRATION_FS_HISTORY_FILE.toFile(), history);
+ }
+
+ protected void setHttpClient(CloseableHttpClient httpClient) {
+ this.httpClient = httpClient;
+ }
+
+ /**
+ * A simple migration step to be performed
+ */
+ public interface MigrationStep {
+ /**
+ * Do you migration a safe and unitary way, so that in case this step fail it can be re-executed safely
+ */
+ void execute() throws Exception;
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationScript.java
similarity index 90%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationScript.java
index 15533f056..df3eab2ce 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationScript.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
import groovy.lang.Script;
import org.apache.commons.io.IOUtils;
@@ -48,7 +48,7 @@ public class MigrationScript implements Comparable<MigrationScript> {
private final int priority;
private final String name;
- public MigrationScript(URL scriptURL, Bundle bundle) throws IOException {
+ protected MigrationScript(URL scriptURL, Bundle bundle) throws IOException {
this.bundle = bundle;
this.script = IOUtils.toString(scriptURL);
@@ -65,31 +65,31 @@ public class MigrationScript implements Comparable<MigrationScript> {
}
}
- public Script getCompiledScript() {
+ protected Script getCompiledScript() {
return compiledScript;
}
- public void setCompiledScript(Script compiledScript) {
+ protected void setCompiledScript(Script compiledScript) {
this.compiledScript = compiledScript;
}
- public String getScript() {
+ protected String getScript() {
return script;
}
- public Bundle getBundle() {
+ protected Bundle getBundle() {
return bundle;
}
- public Version getVersion() {
+ protected Version getVersion() {
return version;
}
- public int getPriority() {
+ protected int getPriority() {
return priority;
}
- public String getName() {
+ protected String getName() {
return name;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationServiceImpl.java
similarity index 58%
copy from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
copy to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationServiceImpl.java
index a21b5d8c3..9722bb3fd 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationServiceImpl.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
import groovy.lang.GroovyClassLoader;
import groovy.lang.GroovyShell;
@@ -25,16 +25,18 @@ import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.action.Action;
-import org.apache.karaf.shell.api.action.Argument;
-import org.apache.karaf.shell.api.action.Command;
-import org.apache.karaf.shell.api.action.lifecycle.Reference;
-import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.MigrationService;
import org.apache.unomi.shell.migration.utils.HttpUtils;
-import org.osgi.framework.*;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.Version;
import org.osgi.framework.wiring.BundleWiring;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
import java.io.IOException;
import java.net.URL;
@@ -42,113 +44,121 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.unomi.shell.migration.actions.MigrationConfig.*;
+import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
-@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " +
- "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process.")
-@Service
-public class Migrate implements Action {
+@Component(service = MigrationService.class, immediate = true)
+public class MigrationServiceImpl implements MigrationService {
- protected static final String MIGRATION_FS_ROOT_FOLDER = "migration";
- protected static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
+ public static final String MIGRATION_FS_ROOT_FOLDER = "migration";
+ public static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
- @Reference
- Session session;
+ private BundleContext bundleContext;
- @Reference
- BundleContext bundleContext;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ private MigrationConfig migrationConfig;
- @Reference
- MigrationConfig migrationConfig;
-
- @Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
- private String originVersion;
+ @Activate
+ public void activate(ComponentContext componentContext) {
+ this.bundleContext = componentContext.getBundleContext();
+ }
- @Argument(index = 1, name = "skipConfirmation", description = "Should the confirmation before starting the migration process be skipped ?", valueToShowInHelp = "false")
- private boolean skipConfirmation = false;
+ public void migrateUnomi(String originVersion, boolean skipConfirmation, Session session) throws Exception {
+ System.out.println("Migrating Unomi...");
+ // Wait for config to be loaded by file install, in case of unomi.autoMigrate the OSGI conf may take a few seconds to be loaded correctly.
+ waitForMigrationConfigLoad(60, 1);
- public Object execute() throws Exception {
// Load migration scrips
Set<MigrationScript> scripts = loadOSGIScripts();
scripts.addAll(loadFileSystemScripts());
+ // Create migration context
+ Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
+ MigrationContext context = new MigrationContext(session, migrationConfig);
+ context.tryRecoverFromHistory();
+
+ // no origin version, just print available scripts
if (originVersion == null) {
- displayMigrations(scripts);
- ConsoleUtils.printMessage(session, "Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
- return null;
+ displayMigrations(scripts, context);
+ context.printMessage("Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
+ return;
}
// Check that there is some migration scripts available from given version
Version fromVersion = new Version(originVersion);
scripts = filterScriptsFromVersion(scripts, fromVersion);
if (scripts.size() == 0) {
- ConsoleUtils.printMessage(session, "No migration scripts available found starting from version: " + originVersion);
- return null;
+ context.printMessage("No migration scripts available found starting from version: " + originVersion);
+ return;
} else {
- ConsoleUtils.printMessage(session, "The following migration scripts starting from version: " + originVersion + " will be executed.");
- displayMigrations(scripts);
+ context.printMessage("The following migration scripts starting from version: " + originVersion + " will be executed.");
+ displayMigrations(scripts, context);
}
// Check for user approval before migrate
- if (!skipConfirmation && ConsoleUtils.askUserWithAuthorizedAnswer(session,
+ if (!skipConfirmation && context.askUserWithAuthorizedAnswer(
"[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ",
Arrays.asList("yes", "no")).equalsIgnoreCase("no")) {
- ConsoleUtils.printMessage(session, "Migration process aborted");
- return null;
+ context.printMessage("Migration process aborted");
+ return;
}
- // reset migration config from previous stored users choices.
- migrationConfig.reset();
- Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
- MigrationHistory migrationHistory = new MigrationHistory(session, migrationConfig);
- migrationHistory.tryRecover();
-
// Handle credentials
CredentialsProvider credentialsProvider = null;
- String login = migrationConfig.getString(CONFIG_ES_LOGIN, session);
+ String login = context.getConfigString(CONFIG_ES_LOGIN);
if (StringUtils.isNotEmpty(login)) {
credentialsProvider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials
- = new UsernamePasswordCredentials(login, migrationConfig.getString(CONFIG_ES_PASSWORD, session));
+ = new UsernamePasswordCredentials(login, context.getConfigString(CONFIG_ES_PASSWORD));
credentialsProvider.setCredentials(AuthScope.ANY, credentials);
}
- try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session), credentialsProvider)) {
+ try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(context.getConfigBoolean(CONFIG_TRUST_ALL_CERTIFICATES), credentialsProvider)) {
// Compile scripts
- scripts = parseScripts(scripts, session, httpClient, migrationConfig, migrationHistory);
+ context.setHttpClient(httpClient);
+ scripts = parseScripts(scripts, context);
// Start migration
- ConsoleUtils.printMessage(session, "Starting migration process from version: " + originVersion);
+ context.printMessage("Starting migration process from version: " + originVersion);
for (MigrationScript migrateScript : scripts) {
- ConsoleUtils.printMessage(session, "Starting execution of: " + migrateScript);
+ context.printMessage("Starting execution of: " + migrateScript);
try {
migrateScript.getCompiledScript().run();
} catch (Exception e) {
- ConsoleUtils.printException(session, "Error executing: " + migrateScript, e);
- return null;
+ context.printException("Error executing: " + migrateScript, e);
+ throw e;
}
- ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
+ context.printMessage("Finish execution of: " + migrateScript);
}
- // We clean history, migration is successful
- migrationHistory.clean();
+ // Persist final flag in history
+ context.performMigrationStep("migrationStatus", () -> { /* nothing it's just a marker to persist in the history to know that everything is finished */ });
}
+ }
- return null;
+ private void waitForMigrationConfigLoad(int maxTry, int secondsToSleep) throws InterruptedException {
+ while (!migrationConfig.getConfig().containsKey("felix.fileinstall.filename")) {
+ maxTry -= 1;
+ if (maxTry == 0) {
+ throw new IllegalStateException("Waited too long for migration config to be available");
+ } else {
+ TimeUnit.SECONDS.sleep(secondsToSleep);
+ }
+ }
}
- private void displayMigrations(Set<MigrationScript> scripts) {
+ private void displayMigrations(Set<MigrationScript> scripts, MigrationContext context) {
Version previousVersion = new Version("0.0.0");
for (MigrationScript migration : scripts) {
if (migration.getVersion().getMajor() > previousVersion.getMajor() || migration.getVersion().getMinor() > previousVersion.getMinor()) {
- ConsoleUtils.printMessage(session, "From " + migration.getVersion().getMajor() + "." + migration.getVersion().getMinor() + ".0:");
+ context.printMessage("From " + migration.getVersion().getMajor() + "." + migration.getVersion().getMinor() + ".0:");
}
- ConsoleUtils.printMessage(session, "- " + migration);
+ context.printMessage("- " + migration);
previousVersion = migration.getVersion();
}
}
@@ -159,7 +169,7 @@ public class Migrate implements Action {
.collect(Collectors.toCollection(TreeSet::new));
}
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
+ private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, MigrationContext context) {
Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
return scripts.stream()
@@ -167,7 +177,7 @@ public class Migrate implements Action {
// fallback on current bundle if the scripts is not provided by OSGI
Bundle scriptBundle = migrateScript.getBundle() != null ? migrateScript.getBundle() : bundleContext.getBundle();
if (!shellsPerBundle.containsKey(scriptBundle.getSymbolicName())) {
- shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig, migrationHistory));
+ shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, context));
}
migrateScript.setCompiledScript(shellsPerBundle.get(scriptBundle.getSymbolicName()).parse(migrateScript.getScript()));
})
@@ -212,15 +222,16 @@ public class Migrate implements Action {
return migrationScripts;
}
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
+ private GroovyShell buildShellForBundle(Bundle bundle, MigrationContext context) {
GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
- groovyShell.setVariable("session", session);
- groovyShell.setVariable("httpClient", httpClient);
- groovyShell.setVariable("migrationConfig", migrationConfig);
- groovyShell.setVariable("migrationHistory", migrationHistory);
+ groovyShell.setVariable("migrationContext", context);
groovyShell.setVariable("bundleContext", bundle.getBundleContext());
return groovyShell;
}
+
+ public void setMigrationConfig(MigrationConfig migrationConfig) {
+ this.migrationConfig = migrationConfig;
+ }
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
deleted file mode 100644
index b56a7568c..000000000
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.shell.migration.utils;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.karaf.shell.api.console.Session;
-import org.jline.reader.LineReader;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-
-/**
- * @author dgaillard
- */
-public class ConsoleUtils {
-
- /**
- * This will ask a question to the user and return the default answer if the user does not answer.
- *
- * @param session the shell's session
- * @param msg String message to ask
- * @param defaultAnswer String default answer
- * @return the user's answer
- * @throws IOException if there was a problem reading input from the console
- */
- public static String askUserWithDefaultAnswer(Session session, String msg, String defaultAnswer) throws IOException {
- String answer = promptMessageToUser(session, msg);
- if (StringUtils.isBlank(answer)) {
- return defaultAnswer;
- }
- return answer;
- }
-
- /**
- * This method allow you to ask a question to the user.
- * The answer is controlled before being return so the question will be ask until the user enter one the authorized answer
- *
- * @param session the shell's session
- * @param msg String message to ask
- * @param authorizedAnswer Array of possible answer, all answer must be in lower case
- * @return the user answer
- * @throws IOException if there was an error retrieving an answer from the user on the console
- */
- public static String askUserWithAuthorizedAnswer(Session session, String msg, List<String> authorizedAnswer) throws IOException {
- String answer;
- do {
- answer = promptMessageToUser(session,msg);
- } while (!authorizedAnswer.contains(answer.toLowerCase()));
- return answer;
- }
-
- /**
- * This method allow you to prompt a message to the user.
- * No control is done on the answer provided by the user.
- *
- * @param session the shell's session
- * @param msg String message to prompt
- * @return the user answer
- */
- public static String promptMessageToUser(Session session, String msg) {
- LineReader reader = (LineReader) session.get(".jline.reader");
- return reader.readLine(msg, null);
- }
-
- /**
- * Print a message in the console.
- * @param session the shell's session
- * @param msg the message to print out with a newline
- */
- public static void printMessage(Session session, String msg) {
- PrintStream writer = session.getConsole();
- writer.println(msg);
- }
-
- /**
- * Print an exception along with a message in the console.
- * @param session the shell's session
- * @param msg the message to print out with a newline
- * @param t the exception to dump in the shell console after the message
- */
- public static void printException(Session session, String msg, Throwable t) {
- PrintStream writer = session.getConsole();
- writer.println(msg);
- t.printStackTrace(writer);
- }
-}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 9840b0184..d44be389c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -23,7 +23,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
-import org.apache.unomi.shell.migration.actions.MigrationHistory;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.json.JSONArray;
import org.json.JSONObject;
import org.osgi.framework.Bundle;
@@ -142,7 +142,7 @@ public class MigrationUtils {
}
public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
- String newIndexSettings, String painlessScript, MigrationHistory history) throws Exception {
+ String newIndexSettings, String painlessScript, MigrationContext migrationContext) throws Exception {
if (indexName.endsWith("-cloned")) {
// We should never reIndex a clone ...
return;
@@ -156,7 +156,7 @@ public class MigrationUtils {
String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json");
- history.performMigrationStep("Reindex step for: " + indexName + " (clone creation)", () -> {
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (clone creation)", () -> {
// Delete clone in case it already exists, could be incomplete from a previous reindex attempt, so better create a fresh one.
if (indexExists(httpClient, esAddress, indexNameCloned)) {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
@@ -167,7 +167,7 @@ public class MigrationUtils {
HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_clone/" + indexNameCloned, null, null);
});
- history.performMigrationStep("Reindex step for: " + indexName + " (recreate the index and perform the re-indexation)", () -> {
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (recreate the index and perform the re-indexation)", () -> {
// Delete original index if it still exists
if (indexExists(httpClient, esAddress, indexName)) {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
@@ -178,7 +178,7 @@ public class MigrationUtils {
HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
});
- history.performMigrationStep("Reindex step for: " + indexName + " (delete clone)", () -> {
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (delete clone)", () -> {
// Delete original index if it still exists
if (indexExists(httpClient, esAddress, indexNameCloned)) {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
index 1619d275c..759cd09f8 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
@@ -17,10 +17,16 @@
package org.apache.unomi.shell.services.internal;
import org.apache.commons.lang3.StringUtils;
+import org.apache.unomi.shell.migration.MigrationService;
import org.apache.unomi.shell.services.UnomiManagementService;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
import org.osgi.framework.BundleException;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
import java.util.ArrayList;
import java.util.Collections;
@@ -29,16 +35,28 @@ import java.util.List;
/**
* @author dgaillard
*/
+@Component(service = UnomiManagementService.class, immediate = true)
public class UnomiManagementServiceImpl implements UnomiManagementService {
private BundleContext bundleContext;
- private List<String> bundleSymbolicNames;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ private MigrationService migrationService;
+
+ private final List<String> bundleSymbolicNames = new ArrayList<>();
private List<String> reversedBundleSymbolicNames;
- public void init() throws BundleException {
+ @Activate
+ public void init(ComponentContext componentContext) throws Exception {
+ this.bundleContext = componentContext.getBundleContext();
initReversedBundleSymbolicNames();
- if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) && bundleContext.getProperty("unomi.autoStart")
- .equals("true")) {
+
+ if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoMigrate"))) {
+ migrationService.migrateUnomi(bundleContext.getProperty("unomi.autoMigrate"), true, null);
+ }
+
+ if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) &&
+ bundleContext.getProperty("unomi.autoStart").equals("true")) {
startUnomi();
}
}
@@ -71,20 +89,43 @@ public class UnomiManagementServiceImpl implements UnomiManagementService {
}
}
- public void setBundleContext(BundleContext bundleContext) {
- this.bundleContext = bundleContext;
- }
-
- public void setBundleSymbolicNames(List<String> bundleSymbolicNames) {
- this.bundleSymbolicNames = bundleSymbolicNames;
- }
-
public void initReversedBundleSymbolicNames() {
+ bundleSymbolicNames.clear();
+ bundleSymbolicNames.add("org.apache.unomi.lifecycle-watcher");
+ bundleSymbolicNames.add("org.apache.unomi.api");
+ bundleSymbolicNames.add("org.apache.unomi.common");
+ bundleSymbolicNames.add("org.apache.unomi.scripting");
+ bundleSymbolicNames.add("org.apache.unomi.metrics");
+ bundleSymbolicNames.add("org.apache.unomi.persistence-spi");
+ bundleSymbolicNames.add("org.apache.unomi.persistence-elasticsearch-core");
+ bundleSymbolicNames.add("org.apache.unomi.services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-rest");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-geonames-services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-geonames-rest");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-privacy-extension-services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-privacy-extension-rest");
+ bundleSymbolicNames.add("org.apache.unomi.json-schema-services");
+ bundleSymbolicNames.add("org.apache.unomi.json-schema-rest");
+ bundleSymbolicNames.add("org.apache.unomi.rest");
+ bundleSymbolicNames.add("org.apache.unomi.wab");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-base");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-request");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-mail");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-optimization-test");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-actions");
+ bundleSymbolicNames.add("org.apache.unomi.router-api");
+ bundleSymbolicNames.add("org.apache.unomi.router-core");
+ bundleSymbolicNames.add("org.apache.unomi.router-service");
+ bundleSymbolicNames.add("org.apache.unomi.router-rest");
+ bundleSymbolicNames.add("org.apache.unomi.shell-dev-commands");
+ bundleSymbolicNames.add("org.apache.unomi.groovy-actions-services");
+ bundleSymbolicNames.add("org.apache.unomi.groovy-actions-rest");
+
if (reversedBundleSymbolicNames == null || reversedBundleSymbolicNames.isEmpty()) {
this.reversedBundleSymbolicNames = new ArrayList<>();
reversedBundleSymbolicNames.addAll(bundleSymbolicNames);
Collections.reverse(reversedBundleSymbolicNames);
}
}
-
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy
index 152f432a1..45f7b6fd5 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy
@@ -17,4 +17,4 @@ import org.apache.unomi.shell.migration.impl.MigrationTo121
* limitations under the License.
*/
-new MigrationTo121().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
+new MigrationTo121().execute(migrationContext, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy
index 99fbd8f66..c9530a646 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy
@@ -17,4 +17,4 @@ import org.apache.unomi.shell.migration.impl.MigrationTo122
* limitations under the License.
*/
-new MigrationTo122().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
+new MigrationTo122().execute(migrationContext, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy
index 4fbf01423..7151b2f3b 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy
@@ -17,4 +17,4 @@ import org.apache.unomi.shell.migration.impl.MigrationTo150
* limitations under the License.
*/
-new MigrationTo150().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
+new MigrationTo150().execute(migrationContext, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index 9dc340405..dcc2a0511 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -1,7 +1,5 @@
import groovy.json.JsonSlurper
-import org.apache.http.impl.client.CloseableHttpClient
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -25,33 +23,32 @@ import java.time.Instant
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-CloseableHttpClient client = httpClient
+MigrationContext context = migrationContext
Instant migrationTime = Instant.now()
def jsonSlurper = new JsonSlurper()
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String aliasIndex = indexPrefix + "-profilealias"
String profileIndex = indexPrefix + "-profile"
-history.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
- if (!MigrationUtils.indexExists(client, esAddress, aliasIndex)) {
+context.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, aliasIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, profileIndex, mapping)
- HttpUtils.executePutRequest(client, esAddress + "/" + aliasIndex, newIndexSettings, null)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, profileIndex, mapping)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + aliasIndex, newIndexSettings, null)
}
})
-history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles", () -> {
+context.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles", () -> {
String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
String profileMergedSearchRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_search.json")
- MigrationUtils.scrollQuery(client, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", hits -> {
+ MigrationUtils.scrollQuery(context.getHttpClient(), esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", hits -> {
// create aliases for those merged profiles and delete them.
def jsonHits = jsonSlurper.parseText(hits)
- ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " existing profiles merged")
+ context.printMessage("Detected: " + jsonHits.size() + " existing profiles merged")
final StringBuilder bulkSaveRequest = new StringBuilder()
jsonHits.each {
@@ -63,14 +60,14 @@ history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles"
def aliasAlreadyExists = false
try {
- def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + profileIndex + "/_doc/" + masterProfileId, null))
+ def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(context.getHttpClient(), esAddress + "/" + profileIndex + "/_doc/" + masterProfileId, null))
masterProfileExists = masterProfile.found
} catch (HttpRequestException e) {
// can happen in case response code > 400 due to item not exist in ElasticSearch
}
try {
- def existingAlias = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + aliasIndex + "/_doc/" + mergedProfileId, null));
+ def existingAlias = jsonSlurper.parseText(HttpUtils.executeGetRequest(context.getHttpClient(), esAddress + "/" + aliasIndex + "/_doc/" + mergedProfileId, null));
aliasAlreadyExists = existingAlias.found
} catch (HttpRequestException e) {
// can happen in case of response code > 400 due to item not exist in ElasticSearch
@@ -86,12 +83,12 @@ history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles"
}
if (bulkSaveRequest.length() > 0) {
- HttpUtils.executePostRequest(client, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
})
})
-history.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
+context.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
- HttpUtils.executePostRequest(client, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 273389cff..3f46f0dd6 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -1,7 +1,5 @@
import groovy.json.JsonSlurper
-import org.apache.http.impl.client.CloseableHttpClient
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -23,28 +21,27 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-CloseableHttpClient client = httpClient
+MigrationContext context = migrationContext
def jsonSlurper = new JsonSlurper()
String searchScopesRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/scope_search.json")
String saveScopeRequestBulk = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scope_save_bulk.ndjson")
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String scopeIndex = indexPrefix + "-scope"
-history.performMigrationStep("2.0.0-create-scope-index", () -> {
- if (!MigrationUtils.indexExists(client, esAddress, scopeIndex)) {
+context.performMigrationStep("2.0.0-create-scope-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, indexPrefix + "-profile", mapping)
- HttpUtils.executePutRequest(client, esAddress + "/" + scopeIndex, newIndexSettings, null)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, "${indexPrefix}-profile", mapping)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null)
}
})
-history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
+context.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
// search existing scopes from event
- def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(client, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
- ConsoleUtils.printMessage(session, "Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
+ def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
+ context.printMessage("Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
// create scopes
def buckets = searchResponse.aggregations.scopes.buckets
@@ -58,7 +55,7 @@ history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
// check that the scope doesn't already exists
def scopeAlreadyExists = false
try {
- def existingScope = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + scopeIndex + "/_doc/" + bucket.key, null));
+ def existingScope = jsonSlurper.parseText(HttpUtils.executeGetRequest(context.getHttpClient(), esAddress + "/" + scopeIndex + "/_doc/" + bucket.key, null));
scopeAlreadyExists = existingScope.found
} catch (HttpRequestException e) {
// can happen in case response code > 400 due to item not exist in ElasticSearch
@@ -72,7 +69,7 @@ history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
}
if (bulkSaveRequest.length() > 0) {
- HttpUtils.executePostRequest(client, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
}
})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 4d1df4791..826243ae4 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -1,4 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -18,16 +18,14 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"];
indicesToReindex.each { indexToReindex ->
- String mappingFileName = "${indexToReindex}.json"
- String realIndexName = "${indexPrefix}-${indexToReindex.toLowerCase()}"
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, mappingFileName)
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, realIndexName, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, realIndexName, newIndexSettings, null, history)
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "${indexToReindex}.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-profile", mapping)
+ MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReindex.toLowerCase()}", newIndexSettings, null, context)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index b0bc12d6e..4711056c0 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -1,4 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -18,12 +18,12 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, indexPrefix + "-profile", mapping)
-MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), history)
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-segment", mapping)
+MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, indexPrefix + "-profile",
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), context)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index 492c7ae1a..941ff82a8 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -1,4 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -19,21 +19,23 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
-history.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
+context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
- HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+ HttpUtils.executePostRequest(context.getHttpClient(), "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
})
// Reindex the rest of the events
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
-Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, "${indexPrefix}-event-")
+Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-")
+// use session indices to extract monthly index settings
+Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-session-")
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, sessionIndices[0], mapping)
eventIndices.each { eventIndex ->
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, eventIndex, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, history)
+ MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context)
}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/tools/shell-commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml
deleted file mode 100644
index 61f70c335..000000000
--- a/tools/shell-commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<blueprint xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
- xsi:schemaLocation="http://www.osgi.org/xmlns/blueprint/v1.0.0 http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd">
-
- <bean id="unomiManagementServiceImpl" class="org.apache.unomi.shell.services.internal.UnomiManagementServiceImpl" init-method="init">
- <property name="bundleSymbolicNames">
- <list>
- <value>org.apache.unomi.lifecycle-watcher</value>
- <value>org.apache.unomi.api</value>
- <value>org.apache.unomi.common</value>
- <value>org.apache.unomi.scripting</value>
- <value>org.apache.unomi.metrics</value>
- <value>org.apache.unomi.persistence-spi</value>
- <value>org.apache.unomi.persistence-elasticsearch-core</value>
- <value>org.apache.unomi.services</value>
- <value>org.apache.unomi.cxs-lists-extension-services</value>
- <value>org.apache.unomi.cxs-lists-extension-rest</value>
- <value>org.apache.unomi.cxs-geonames-services</value>
- <value>org.apache.unomi.cxs-geonames-rest</value>
- <value>org.apache.unomi.cxs-privacy-extension-services</value>
- <value>org.apache.unomi.cxs-privacy-extension-rest</value>
- <value>org.apache.unomi.json-schema-services</value>
- <value>org.apache.unomi.json-schema-rest</value>
- <value>org.apache.unomi.rest</value>
- <value>org.apache.unomi.wab</value>
- <value>org.apache.unomi.plugins-base</value>
- <value>org.apache.unomi.plugins-request</value>
- <value>org.apache.unomi.plugins-mail</value>
- <value>org.apache.unomi.plugins-optimization-test</value>
- <value>org.apache.unomi.cxs-lists-extension-actions</value>
- <value>org.apache.unomi.router-api</value>
- <value>org.apache.unomi.router-core</value>
- <value>org.apache.unomi.router-service</value>
- <value>org.apache.unomi.router-rest</value>
- <value>org.apache.unomi.shell-dev-commands</value>
- <value>org.apache.unomi.groovy-actions-services</value>
- <value>org.apache.unomi.groovy-actions-rest</value>
- </list>
- </property>
- <property name="bundleContext" ref="blueprintBundleContext"/>
- </bean>
- <service id="unomiManagementService" ref="unomiManagementServiceImpl"
- interface="org.apache.unomi.shell.services.UnomiManagementService"/>
-</blueprint>
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index b5dda03ac..5749e4cf9 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -18,12 +18,13 @@
# Migration config used for silent migration
# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
-# esAddress = http://localhost:9200
-# esLogin = elastic
-# esPassword = password
-# httpClient.trustAllCertificates = true
-# indexPrefix = context
+esAddresses = ${org.apache.unomi.elasticsearch.addresses:-localhost:9200}
+esSSLEnabled = ${org.apache.unomi.elasticsearch.sslEnable:-false}
+esLogin = ${org.apache.unomi.elasticsearch.username:-}
+esPassword = ${org.apache.unomi.elasticsearch.password:-}
+httpClient.trustAllCertificates = ${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false}
+indexPrefix = ${org.apache.unomi.elasticsearch.index.prefix:-context}
# Should the migration try to recover from a previous run ?
# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
-# recoverFromHistory = true
\ No newline at end of file
+recoverFromHistory = ${org.apache.unomi.migration.recoverFromHistory:-true}
\ No newline at end of file