You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/16 21:56:10 UTC
[unomi] branch migrationAutomation created (now f5801310e)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a change to branch migrationAutomation
in repository https://gitbox.apache.org/repos/asf/unomi.git
at f5801310e UNOMI-622: provide automation support for migration
This branch includes the following new commits:
new f5801310e UNOMI-622: provide automation support for migration
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-622: provide automation support for migration
Posted by jk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch migrationAutomation
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit f5801310eb59f90ab25a5638531a5226cfcf60ed
Author: Kevan <ke...@jahia.com>
AuthorDate: Tue Aug 16 23:55:46 2022 +0200
UNOMI-622: provide automation support for migration
---
.../test/java/org/apache/unomi/itests/BaseIT.java | 1 -
.../apache/unomi/itests/migration/MigrationIT.java | 15 +-
.../migrate-11.0.0-01-failingMigration.groovy | 25 +-
.../migrate-11.0.0-01-successMigration.groovy | 25 +-
.../migration/org.apache.unomi.migration.cfg | 28 ---
.../main/resources/etc/custom.system.properties | 5 +
tools/shell-commands/pom.xml | 2 +-
.../apache/unomi/shell/migration/Migration.java | 11 +-
.../unomi/shell/migration/MigrationService.java | 38 +++
.../unomi/shell/migration/actions/Migrate.java | 187 +-------------
.../shell/migration/actions/MigrationHistory.java | 124 ---------
.../unomi/shell/migration/impl/MigrationTo121.java | 21 +-
.../unomi/shell/migration/impl/MigrationTo122.java | 23 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 52 ++--
.../{actions => service}/MigrationConfig.java | 58 +----
.../MigrationConfigProperty.java | 4 +-
.../shell/migration/service/MigrationContext.java | 280 +++++++++++++++++++++
.../{actions => service}/MigrationScript.java | 18 +-
.../MigrationServiceImpl.java} | 139 +++++-----
.../unomi/shell/migration/utils/ConsoleUtils.java | 101 --------
.../shell/migration/utils/MigrationUtils.java | 10 +-
.../internal/UnomiManagementServiceImpl.java | 63 ++++-
.../migration/migrate-1.2.1-00-migrateTags.groovy | 2 +-
.../migrate-1.2.2-00-deleteOldIndexTemplate.groovy | 2 +-
.../migrate-1.5.0-00-elasticSearch7.4.groovy | 2 +-
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 35 ++-
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 29 +--
.../migrate-2.0.0-05-globalReindex.groovy | 12 +-
.../migrate-2.0.0-10-profileReindex.groovy | 14 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 18 +-
.../resources/OSGI-INF/blueprint/blueprint.xml | 62 -----
.../main/resources/org.apache.unomi.migration.cfg | 13 +-
32 files changed, 622 insertions(+), 797 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
index dc103951d..f1bb826b7 100644
--- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java
@@ -235,7 +235,6 @@ public abstract class BaseIT extends KarafTestSupport {
System.out.println("==== Configuring container");
Option[] options = new Option[]{
replaceConfigurationFile("etc/org.apache.unomi.router.cfg", new File("src/test/resources/org.apache.unomi.router.cfg")),
- replaceConfigurationFile("etc/org.apache.unomi.migration.cfg", new File("src/test/resources/migration/org.apache.unomi.migration.cfg")),
replaceConfigurationFile("data/tmp/1-basic-test.csv", new File("src/test/resources/1-basic-test.csv")),
replaceConfigurationFile("data/tmp/recurrent_import/2-surfers-test.csv", new File("src/test/resources/2-surfers-test.csv")),
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
index 94f546c01..a6f554c4a 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/MigrationIT.java
@@ -40,16 +40,11 @@ public class MigrationIT extends BaseIT {
Files.createDirectories(BASE_DIRECTORIES);
Files.write(FAILING_SCRIPT_FS_PATH, bundleResourceAsString(FAILING_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
- String failingResult = executeCommand("unomi:migrate 10.0.0 true");
- System.out.println("Intentional failing migration result:");
- System.out.println(failingResult);
- // step 4 and 5 should not be contains, step 3 is failing
- // Only step 1, 2 and 3 should be performed.
- Assert.assertTrue(failingResult.contains("inside step 1"));
- Assert.assertTrue(failingResult.contains("inside step 2"));
- Assert.assertTrue(failingResult.contains("inside step 3"));
- Assert.assertTrue(!failingResult.contains("inside step 4"));
- Assert.assertTrue(!failingResult.contains("inside step 5"));
+ try {
+ executeCommand("unomi:migrate 10.0.0 true");
+ } catch (Exception e) {
+ // this is expected, the script fail at step 3
+ }
Files.deleteIfExists(FAILING_SCRIPT_FS_PATH);
Files.write(SUCCESS_SCRIPT_FS_PATH, bundleResourceAsString(SUCCESS_SCRIPT_RESOURCE).getBytes(StandardCharsets.UTF_8));
diff --git a/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
index 84ea5e908..3e801d725 100644
--- a/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-failingMigration.groovy
@@ -1,5 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,24 +17,24 @@ import org.apache.unomi.shell.migration.utils.ConsoleUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-history.performMigrationStep("step 1", () -> {
- ConsoleUtils.printMessage(session, "inside step 1")
+MigrationContext context = migrationContext
+context.performMigrationStep("step 1", () -> {
+ context.printMessage("inside step 1")
})
-history.performMigrationStep("step 2", () -> {
- ConsoleUtils.printMessage(session, "inside step 2")
+context.performMigrationStep("step 2", () -> {
+ context.printMessage("inside step 2")
})
-history.performMigrationStep("step 3", () -> {
- ConsoleUtils.printMessage(session, "inside step 3")
+context.performMigrationStep("step 3", () -> {
+ context.printMessage("inside step 3")
throw new RuntimeException("Intentional failure !")
})
-history.performMigrationStep("step 4", () -> {
- ConsoleUtils.printMessage(session, "inside step 4")
+context.performMigrationStep("step 4", () -> {
+ context.printMessage("inside step 4")
})
-history.performMigrationStep("step 5", () -> {
- ConsoleUtils.printMessage(session, "inside step 5")
+context.performMigrationStep("step 5", () -> {
+ context.printMessage("inside step 5")
})
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
index 44c4403ad..93db092b9 100644
--- a/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
+++ b/itests/src/test/resources/migration/migrate-11.0.0-01-successMigration.groovy
@@ -1,5 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,23 +17,23 @@ import org.apache.unomi.shell.migration.utils.ConsoleUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-history.performMigrationStep("step 1", () -> {
- ConsoleUtils.printMessage(session, "inside step 1")
+MigrationContext context = migrationContext
+context.performMigrationStep("step 1", () -> {
+ context.printMessage("inside step 1")
})
-history.performMigrationStep("step 2", () -> {
- ConsoleUtils.printMessage(session, "inside step 2")
+context.performMigrationStep("step 2", () -> {
+ context.printMessage("inside step 2")
})
-history.performMigrationStep("step 3", () -> {
- ConsoleUtils.printMessage(session, "inside step 3")
+context.performMigrationStep("step 3", () -> {
+ context.printMessage("inside step 3")
})
-history.performMigrationStep("step 4", () -> {
- ConsoleUtils.printMessage(session, "inside step 4")
+context.performMigrationStep("step 4", () -> {
+ context.printMessage("inside step 4")
})
-history.performMigrationStep("step 5", () -> {
- ConsoleUtils.printMessage(session, "inside step 5")
+context.performMigrationStep("step 5", () -> {
+ context.printMessage("inside step 5")
})
\ No newline at end of file
diff --git a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg b/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
deleted file mode 100644
index d96120c28..000000000
--- a/itests/src/test/resources/migration/org.apache.unomi.migration.cfg
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Migration config used for silent migration
-
-# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
-esAddress = http://localhost:9400
-esLogin =
-httpClient.trustAllCertificates = true
-indexPrefix = context
-
-# Should the migration try to recover from a previous run ?
-# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
-recoverFromHistory = true
\ No newline at end of file
diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties
index fe09b36b9..d0525ca5d 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -419,3 +419,8 @@ org.apache.unomi.weatherUpdate.url.attributes=${env:UNOMI_WEATHERUPDATE_URL_ATTR
## Settings for GraphQL ##
#######################################################################################################################
org.apache.unomi.graphql.feature.activated=${env:UNOMI_GRAPHQL_FEATURE_ACTIVATED:-false}
+
+#######################################################################################################################
+## Settings for migration ##
+#######################################################################################################################
+org.apache.unomi.migration.recoverFromHistory=${env:UNOMI_MIGRATION_RECOVER_FROM_HISTORY:-true}
\ No newline at end of file
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 0a3df585f..df8b398c9 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -118,7 +118,7 @@
<instructions>
<Export-Package>
org.apache.unomi.shell.migration.utils,
- org.apache.unomi.shell.migration.actions,
+ org.apache.unomi.shell.migration.service,
</Export-Package>
<Embed-Dependency>*;scope=compile|runtime</Embed-Dependency>
<DynamicImport-Package>*</DynamicImport-Package>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
index 0113b3854..b3e103b0b 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
@@ -18,7 +18,8 @@ package org.apache.unomi.shell.migration;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.osgi.framework.BundleContext;
import java.io.IOException;
@@ -30,12 +31,10 @@ import java.io.IOException;
public interface Migration {
/**
* This method is called to execute the migration
- * @param session the shell's session
- * @param httpClient CloseableHttpClient
- * @param migrationConfig config used to perform the migration, like esAddress, trustAllCertificates, etc ...
+ * @param migrationContext the current migration context (config, history, messaging, logging)
* @param bundleContext the bundle context object
* @throws IOException if there was an error while executing the migration
* @deprecated do groovy script for implementing new migrations
*/
- void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException;
-}
+ void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException;
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationService.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationService.java
new file mode 100644
index 000000000..7183ae325
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationService.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration;
+
+import org.apache.karaf.shell.api.console.Session;
+
+/**
+ * Simple OSGI service used for migrating Unomi data
+ */
+public interface MigrationService {
+
+ /**
+ * This will Migrate your data in ES to be compliant with current version.
+ * It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg,
+ * if no configuration is provided then questions will be prompted during the migration process.
+ * (only in case you are in karaf shell context, if not a missing configuration will fail the migration process)
+ *
+ * @param originVersion Origin version without suffix/qualifier (e.g: 1.2.0)
+ * @param skipConfirmation Should the confirmation before starting the migration process be skipped ? (only supported in karaf shell context)
+ * @param session Karaf shell session, for execution in Karaf shell context, null otherwise
+ * @throws Exception
+ */
+ void migrateUnomi(String originVersion, boolean skipConfirmation, Session session) throws Exception;
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index a21b5d8c3..1b5a683e9 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -16,53 +16,25 @@
*/
package org.apache.unomi.shell.migration.actions;
-import groovy.lang.GroovyClassLoader;
-import groovy.lang.GroovyShell;
-import groovy.util.GroovyScriptEngine;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.action.Action;
import org.apache.karaf.shell.api.action.Argument;
import org.apache.karaf.shell.api.action.Command;
import org.apache.karaf.shell.api.action.lifecycle.Reference;
import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
-import org.apache.unomi.shell.migration.utils.HttpUtils;
-import org.osgi.framework.*;
-import org.osgi.framework.wiring.BundleWiring;
-
-import java.io.IOException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static org.apache.unomi.shell.migration.actions.MigrationConfig.*;
+import org.apache.unomi.shell.migration.MigrationService;
@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " +
- "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process.")
+ "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, " +
+ "if no configuration is provided then questions will be prompted during the migration process.")
@Service
public class Migrate implements Action {
- protected static final String MIGRATION_FS_ROOT_FOLDER = "migration";
- protected static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
-
@Reference
Session session;
@Reference
- BundleContext bundleContext;
-
- @Reference
- MigrationConfig migrationConfig;
+ MigrationService migrationService;
@Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
private String originVersion;
@@ -71,156 +43,7 @@ public class Migrate implements Action {
private boolean skipConfirmation = false;
public Object execute() throws Exception {
- // Load migration scrips
- Set<MigrationScript> scripts = loadOSGIScripts();
- scripts.addAll(loadFileSystemScripts());
-
- if (originVersion == null) {
- displayMigrations(scripts);
- ConsoleUtils.printMessage(session, "Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
- return null;
- }
-
- // Check that there is some migration scripts available from given version
- Version fromVersion = new Version(originVersion);
- scripts = filterScriptsFromVersion(scripts, fromVersion);
- if (scripts.size() == 0) {
- ConsoleUtils.printMessage(session, "No migration scripts available found starting from version: " + originVersion);
- return null;
- } else {
- ConsoleUtils.printMessage(session, "The following migration scripts starting from version: " + originVersion + " will be executed.");
- displayMigrations(scripts);
- }
-
- // Check for user approval before migrate
- if (!skipConfirmation && ConsoleUtils.askUserWithAuthorizedAnswer(session,
- "[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ",
- Arrays.asList("yes", "no")).equalsIgnoreCase("no")) {
- ConsoleUtils.printMessage(session, "Migration process aborted");
- return null;
- }
-
- // reset migration config from previous stored users choices.
- migrationConfig.reset();
- Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
- MigrationHistory migrationHistory = new MigrationHistory(session, migrationConfig);
- migrationHistory.tryRecover();
-
- // Handle credentials
- CredentialsProvider credentialsProvider = null;
- String login = migrationConfig.getString(CONFIG_ES_LOGIN, session);
- if (StringUtils.isNotEmpty(login)) {
- credentialsProvider = new BasicCredentialsProvider();
- UsernamePasswordCredentials credentials
- = new UsernamePasswordCredentials(login, migrationConfig.getString(CONFIG_ES_PASSWORD, session));
- credentialsProvider.setCredentials(AuthScope.ANY, credentials);
- }
-
- try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session), credentialsProvider)) {
-
- // Compile scripts
- scripts = parseScripts(scripts, session, httpClient, migrationConfig, migrationHistory);
-
- // Start migration
- ConsoleUtils.printMessage(session, "Starting migration process from version: " + originVersion);
- for (MigrationScript migrateScript : scripts) {
- ConsoleUtils.printMessage(session, "Starting execution of: " + migrateScript);
- try {
- migrateScript.getCompiledScript().run();
- } catch (Exception e) {
- ConsoleUtils.printException(session, "Error executing: " + migrateScript, e);
- return null;
- }
-
- ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
- }
-
- // We clean history, migration is successful
- migrationHistory.clean();
- }
-
+ migrationService.migrateUnomi(originVersion, skipConfirmation, session);
return null;
}
-
- private void displayMigrations(Set<MigrationScript> scripts) {
- Version previousVersion = new Version("0.0.0");
- for (MigrationScript migration : scripts) {
- if (migration.getVersion().getMajor() > previousVersion.getMajor() || migration.getVersion().getMinor() > previousVersion.getMinor()) {
- ConsoleUtils.printMessage(session, "From " + migration.getVersion().getMajor() + "." + migration.getVersion().getMinor() + ".0:");
- }
- ConsoleUtils.printMessage(session, "- " + migration);
- previousVersion = migration.getVersion();
- }
- }
-
- private Set<MigrationScript> filterScriptsFromVersion(Set<MigrationScript> scripts, Version fromVersion) {
- return scripts.stream()
- .filter(migrateScript -> fromVersion.compareTo(migrateScript.getVersion()) < 0)
- .collect(Collectors.toCollection(TreeSet::new));
- }
-
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
- Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
-
- return scripts.stream()
- .peek(migrateScript -> {
- // fallback on current bundle if the scripts is not provided by OSGI
- Bundle scriptBundle = migrateScript.getBundle() != null ? migrateScript.getBundle() : bundleContext.getBundle();
- if (!shellsPerBundle.containsKey(scriptBundle.getSymbolicName())) {
- shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig, migrationHistory));
- }
- migrateScript.setCompiledScript(shellsPerBundle.get(scriptBundle.getSymbolicName()).parse(migrateScript.getScript()));
- })
- .collect(Collectors.toCollection(TreeSet::new));
- }
-
- private Set<MigrationScript> loadOSGIScripts() throws IOException {
- SortedSet<MigrationScript> migrationScripts = new TreeSet<>();
- for (Bundle bundle : bundleContext.getBundles()) {
- Enumeration<URL> scripts = bundle.findEntries("META-INF/cxs/migration", "*.groovy", true);
- if (scripts != null) {
- // check for shell
-
- while (scripts.hasMoreElements()) {
- URL scriptURL = scripts.nextElement();
- migrationScripts.add(new MigrationScript(scriptURL, bundle));
- }
- }
- }
-
- return migrationScripts;
- }
-
- private Set<MigrationScript> loadFileSystemScripts() throws IOException {
- // check migration folder exists
- if (!Files.isDirectory(MIGRATION_FS_SCRIPTS_FOLDER)) {
- return Collections.emptySet();
- }
-
- List<Path> paths;
- try (Stream<Path> walk = Files.walk(MIGRATION_FS_SCRIPTS_FOLDER)) {
- paths = walk
- .filter(path -> !Files.isDirectory(path))
- .filter(path -> path.toString().toLowerCase().endsWith("groovy"))
- .collect(Collectors.toList());
- }
-
- SortedSet<MigrationScript> migrationScripts = new TreeSet<>();
- for (Path path : paths) {
- migrationScripts.add(new MigrationScript(path.toUri().toURL(), null));
- }
- return migrationScripts;
- }
-
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
- GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
- GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
- GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
- groovyShell.setVariable("session", session);
- groovyShell.setVariable("httpClient", httpClient);
- groovyShell.setVariable("migrationConfig", migrationConfig);
- groovyShell.setVariable("migrationHistory", migrationHistory);
- groovyShell.setVariable("bundleContext", bundle.getBundleContext());
- return groovyShell;
- }
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
deleted file mode 100644
index 833099e4f..000000000
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationHistory.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.shell.migration.actions;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.unomi.shell.migration.actions.Migrate.MIGRATION_FS_ROOT_FOLDER;
-import static org.apache.unomi.shell.migration.actions.MigrationConfig.MIGRATION_HISTORY_RECOVER;
-
-/**
- * This class allow for keeping track of the migration steps by persisting the steps and there state on the FileSystem,
- * allowing for a migration to be able to restart from a failure in case it happens.
- */
-public class MigrationHistory {
-
- private static final Path MIGRATION_FS_HISTORY_FILE = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "history.json");
-
- private enum MigrationStepState {
- STARTED,
- COMPLETED
- }
-
- public MigrationHistory(Session session, MigrationConfig migrationConfig) {
- this.session = session;
- this.migrationConfig = migrationConfig;
- this.objectMapper = new ObjectMapper();
-
- }
-
- private final Session session;
- private final MigrationConfig migrationConfig;
- private final ObjectMapper objectMapper;
-
- private Map<String, MigrationStepState> history = new HashMap<>();
-
- /**
- * Try to recover from a previous run
- * I case we found an existing history we will ask if we want to recover or if we want to restart from the beginning
- * (it is also configurable using the conf: recoverFromHistory)
- */
- protected void tryRecover() throws IOException {
- if (Files.exists(MIGRATION_FS_HISTORY_FILE)) {
- if (migrationConfig.getBoolean(MIGRATION_HISTORY_RECOVER, session)) {
- history = objectMapper.readValue(MIGRATION_FS_HISTORY_FILE.toFile(), new TypeReference<Map<String, MigrationStepState>>() {});
- } else {
- clean();
- }
- }
- }
-
- /**
- * this method allow for migration step execution:
- * - in case the history already contains the given stepKey as COMPLETED, then the step won't be executed
- * - in case the history doesn't contain the given stepKey, then the step will be executed
- * Also this method is keeping track of the history by persisting it on the FileSystem.
- *
- * @param stepKey the key of the given step
- * @param step the step to be performed
- * @throws IOException
- */
- public void performMigrationStep(String stepKey, MigrationStep step) throws Exception {
- if (step == null || stepKey == null) {
- throw new IllegalArgumentException("Migration step and/or key cannot be null");
- }
-
- // check if step already exists in history:
- MigrationStepState stepState = history.get(stepKey);
- if (stepState != MigrationStepState.COMPLETED) {
- updateStep(stepKey, MigrationStepState.STARTED);
- step.execute();
- updateStep(stepKey, MigrationStepState.COMPLETED);
- } else {
- ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " already completed in previous run");
- }
- }
-
- /**
- * Clean history from FileSystem
- * @throws IOException
- */
- protected void clean() throws IOException {
- Files.deleteIfExists(MIGRATION_FS_HISTORY_FILE);
- }
-
- private void updateStep(String stepKey, MigrationStepState stepState) throws IOException {
- ConsoleUtils.printMessage(session, "Migration step: " + stepKey + " reach: " + stepState);
- history.put(stepKey, stepState);
- objectMapper.writeValue(MIGRATION_FS_HISTORY_FILE.toFile(), history);
- }
-
- /**
- * A simple migration step to be performed
- */
- public interface MigrationStep {
- /**
- * Do you migration a safe and unitary way, so that in case this step fail it can be re-executed safely
- */
- void execute() throws Exception;
- }
-}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
index 8c39fadea..f19b55970 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
@@ -18,10 +18,9 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.apache.unomi.shell.migration.utils.MigrationUtils;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -36,23 +35,21 @@ import java.util.*;
public class MigrationTo121 implements Migration {
private CloseableHttpClient httpClient;
- private Session session;
private String esAddress;
private LinkedHashMap<String, List<String>> tagsStructurePriorTo130;
private List propsTaggedAsPersonalIdentifier = Arrays.asList("firstName", "lastName", "email", "phoneNumber", "address", "facebookId", "googleId", "linkedInId", "twitterId");
@Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- this.httpClient = httpClient;
- this.session = session;
- this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- migrateTags();
+ public void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException {
+ this.httpClient = migrationContext.getHttpClient();
+ this.esAddress = migrationContext.getConfigString(MigrationConfig.CONFIG_ES_ADDRESS);
+ migrateTags(migrationContext);
}
- private void migrateTags() throws IOException {
+ private void migrateTags(MigrationContext migrationContext) throws IOException {
initTagsStructurePriorTo130();
- String tagsOperation = ConsoleUtils.askUserWithAuthorizedAnswer(session, "How to manage tags?\n1. copy: will duplicate tags in systemTags property\n2. move: will move tags in systemTags property\n[1 - 2]: ", Arrays.asList("1", "2"));
- String removeNamespaceOnSystemTags = ConsoleUtils.askUserWithAuthorizedAnswer(session, "As we will copy/move the tags, do you wish to remove existing namespace on tags before copy/move in systemTags? (e.g: hidden.) (yes/no): ", Arrays.asList("yes", "no"));
+ String tagsOperation = migrationContext.askUserWithAuthorizedAnswer("How to manage tags?\n1. copy: will duplicate tags in systemTags property\n2. move: will move tags in systemTags property\n[1 - 2]: ", Arrays.asList("1", "2"));
+ String removeNamespaceOnSystemTags = migrationContext.askUserWithAuthorizedAnswer("As we will copy/move the tags, do you wish to remove existing namespace on tags before copy/move in systemTags? (e.g: hidden.) (yes/no): ", Arrays.asList("yes", "no"));
List<String> typeToMigrate = Arrays.asList("actionType", "conditionType", "campaign", "goal", "rule", "scoring", "segment", "userList");
for (String type : typeToMigrate) {
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
index 8fab2ee9d..3fab4c182 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
@@ -17,10 +17,9 @@
package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.apache.unomi.shell.migration.utils.HttpRequestException;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.osgi.framework.BundleContext;
@@ -29,30 +28,26 @@ import java.io.IOException;
public class MigrationTo122 implements Migration {
private CloseableHttpClient httpClient;
- private Session session;
private String esAddress;
@Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- this.httpClient = httpClient;
- this.session = session;
- this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- deleteOldIndexTemplate();
-
+ public void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException {
+ this.httpClient = migrationContext.getHttpClient();
+ this.esAddress = migrationContext.getConfigString(MigrationConfig.CONFIG_ES_ADDRESS);
+ deleteOldIndexTemplate(migrationContext);
}
- private void deleteOldIndexTemplate() throws IOException {
+ private void deleteOldIndexTemplate(MigrationContext migrationContext) throws IOException {
String oldMonthlyIndexTemplate = "context_monthlyindex";
try {
- ConsoleUtils.printMessage(session,"Deleting old monthly index template " + oldMonthlyIndexTemplate);
+ migrationContext.printMessage("Deleting old monthly index template " + oldMonthlyIndexTemplate);
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/_template/" + oldMonthlyIndexTemplate, null);
} catch (HttpRequestException e) {
if (e.getCode() == 404) {
- ConsoleUtils.printMessage(session,"Old monthly index template not found, skipping deletion");
+ migrationContext.printMessage("Old monthly index template not found, skipping deletion");
} else {
throw e;
}
}
-
}
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index c73bdab01..9131ea354 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -17,10 +17,9 @@
package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.actions.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.service.MigrationConfig;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -38,13 +37,14 @@ public class MigrationTo150 implements Migration {
public static final String INDEX_DATE_PREFIX = "date-";
@Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- String esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- String es5Address = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
- String sourceIndexPrefix = migrationConfig.getString(MigrationConfig.INDEX_PREFIX, session);
- String destIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "TARGET index prefix (default: context) : ", "context");
- int numberOfShards = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of shards for TARGET (default: 5) : ", "5"));
- int numberOfReplicas = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of replicas for TARGET (default: 1) : ", "1"));
+ public void execute(MigrationContext migrationContext, BundleContext bundleContext) throws IOException {
+ CloseableHttpClient httpClient = migrationContext.getHttpClient();
+ String esAddress = migrationContext.getConfigString(MigrationConfig.CONFIG_ES_ADDRESS);
+ String es5Address = migrationContext.askUserWithDefaultAnswer("SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
+ String sourceIndexPrefix = migrationContext.getConfigString(MigrationConfig.INDEX_PREFIX);
+ String destIndexPrefix = migrationContext.askUserWithDefaultAnswer("TARGET index prefix (default: context) : ", "context");
+ int numberOfShards = Integer.parseInt(migrationContext.askUserWithDefaultAnswer("Number of shards for TARGET (default: 5) : ", "5"));
+ int numberOfReplicas = Integer.parseInt(migrationContext.askUserWithDefaultAnswer("Number of replicas for TARGET (default: 1) : ", "1"));
Set<String> monthlyIndexTypes = new HashSet<>();
monthlyIndexTypes.add("event");
monthlyIndexTypes.add("session");
@@ -75,27 +75,27 @@ public class MigrationTo150 implements Migration {
if (!monthlyIndexTypes.contains(itemType)) {
String indexName = "geonameEntry".equals(itemType) ? "geonames" : sourceIndexPrefix;
- JSONObject es5TypeMapping = getES5TypeMapping(session, httpClient, es5Address, indexName, itemType);
+ JSONObject es5TypeMapping = getES5TypeMapping(httpClient, es5Address, indexName, itemType);
int es5MappingsTotalFieldsLimit = getES5MappingsTotalFieldsLimit(httpClient, es5Address, indexName);
String destIndexName = itemType.toLowerCase();
if (!indexExists(httpClient, esAddress, destIndexPrefix, destIndexName)) {
createESIndex(httpClient, esAddress, destIndexPrefix, destIndexName, numberOfShards, numberOfReplicas, es5MappingsTotalFieldsLimit, getMergedTypeMapping(es5TypeMapping, newTypeMapping));
- reIndex(session, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
+ reIndex(migrationContext, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
} else {
- ConsoleUtils.printMessage(session, "Index " + getIndexName(destIndexPrefix, itemType.toLowerCase()) + " already exists, skipping re-indexation...");
+ migrationContext.printMessage("Index " + getIndexName(destIndexPrefix, itemType.toLowerCase()) + " already exists, skipping re-indexation...");
}
} else {
for (String indexName : monthlyIndexNames) {
// we need to extract the date part
String datePart = indexName.substring(sourceIndexPrefix.length() + 1);
String destIndexName = itemType.toLowerCase() + "-" + INDEX_DATE_PREFIX + datePart;
- JSONObject es5TypeMapping = getES5TypeMapping(session, httpClient, es5Address, indexName, itemType);
+ JSONObject es5TypeMapping = getES5TypeMapping(httpClient, es5Address, indexName, itemType);
int es5MappingsTotalFieldsLimit = getES5MappingsTotalFieldsLimit(httpClient, es5Address, indexName);
if (!indexExists(httpClient, esAddress, destIndexPrefix, destIndexName)) {
createESIndex(httpClient, esAddress, destIndexPrefix, destIndexName, numberOfShards, numberOfReplicas, es5MappingsTotalFieldsLimit, getMergedTypeMapping(es5TypeMapping, newTypeMapping));
- reIndex(session, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
+ reIndex(migrationContext, httpClient, esAddress, es5Address, indexName, getIndexName(destIndexPrefix, destIndexName), itemType);
} else {
- ConsoleUtils.printMessage(session, "Index " + getIndexName(destIndexPrefix, destIndexName) + " already exists, skipping re-indexation...");
+ migrationContext.printMessage("Index " + getIndexName(destIndexPrefix, destIndexName) + " already exists, skipping re-indexation...");
}
}
}
@@ -103,7 +103,7 @@ public class MigrationTo150 implements Migration {
}
long totalMigrationTime = System.currentTimeMillis() - startTime;
- ConsoleUtils.printMessage(session, "Migration operations completed in " + totalMigrationTime + "ms");
+ migrationContext.printMessage("Migration operations completed in " + totalMigrationTime + "ms");
}
private String loadMappingFile(URL predefinedMappingURL) throws IOException {
@@ -158,7 +158,7 @@ public class MigrationTo150 implements Migration {
HttpUtils.executePutRequest(httpClient, esAddress + "/" + getIndexName(indexPrefix, indexName), indexBody.toString(), null);
}
- private void reIndex(Session session, CloseableHttpClient httpClient,
+ private void reIndex(MigrationContext migrationContext, CloseableHttpClient httpClient,
String esAddress,
String es5Address,
String sourceIndexName,
@@ -176,31 +176,31 @@ public class MigrationTo150 implements Migration {
.put("dest", new JSONObject()
.put("index", destIndexName)
);
- ConsoleUtils.printMessage(session, "Reindexing " + sourceIndexName + " to " + destIndexName + "...");
+ migrationContext.printMessage("Reindexing " + sourceIndexName + " to " + destIndexName + "...");
long startTime = System.currentTimeMillis();
try {
String response = HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reindexSettings.toString(), null);
long reindexationTime = System.currentTimeMillis() - startTime;
- ConsoleUtils.printMessage(session, "Reindexing completed in " + reindexationTime + "ms. Result=" + response);
+ migrationContext.printMessage("Reindexing completed in " + reindexationTime + "ms. Result=" + response);
} catch (IOException ioe) {
- ConsoleUtils.printException(session, "Error executing reindexing", ioe);
- ConsoleUtils.printMessage(session, "Attempting to delete index " + destIndexName + " so that we can restart from this point...");
- deleteIndex(session, httpClient, esAddress, destIndexName);
+ migrationContext.printException("Error executing reindexing", ioe);
+ migrationContext.printMessage("Attempting to delete index " + destIndexName + " so that we can restart from this point...");
+ deleteIndex(migrationContext, httpClient, esAddress, destIndexName);
throw ioe;
}
}
- private void deleteIndex(Session session, CloseableHttpClient httpClient,
+ private void deleteIndex(MigrationContext migrationContext, CloseableHttpClient httpClient,
String esAddress,
String indexName) {
try {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
} catch (IOException ioe) {
- ConsoleUtils.printException(session, "Error attempting to delete index" + indexName, ioe);
+ migrationContext.printException("Error attempting to delete index" + indexName, ioe);
}
}
- private JSONObject getES5TypeMapping(Session session, CloseableHttpClient httpClient, String es5Address, String indexName, String typeName) throws IOException {
+ private JSONObject getES5TypeMapping(CloseableHttpClient httpClient, String es5Address, String indexName, String typeName) throws IOException {
String response = HttpUtils.executeGetRequest(httpClient, es5Address + "/" + indexName, null);
if (response != null) {
JSONObject indexInfo = new JSONObject(response).getJSONObject(indexName);
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
similarity index 60%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
index df01f0de0..33f7f321b 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
@@ -14,30 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
-import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Modified;
-import java.io.IOException;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
- * Service uses to aggregate different configuration needed by the migrations
- * Source of config:
- * - file system in OSGI config file: org.apache.unomi.migration.cfg
- * - user interactions in the console during the migration process
+ * Service uses to provide configuration information for the migration
*/
@Component(immediate = true, service = MigrationConfig.class, configurationPid = {"org.apache.unomi.migration"})
public class MigrationConfig {
public static final String CONFIG_ES_ADDRESS = "esAddress";
+ public static final String CONFIG_ES_ADDRESSES = "esAddresses";
+ public static final String CONFIG_ES_SSL_ENABLED = "esSSLEnabled";
public static final String CONFIG_ES_LOGIN = "esLogin";
public static final String CONFIG_ES_PASSWORD = "esPassword";
public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
@@ -48,10 +43,11 @@ public class MigrationConfig {
public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
- private static final Map<String, MigrationConfigProperty> configProperties;
+ protected static final Map<String, MigrationConfigProperty> configProperties;
static {
Map<String, MigrationConfigProperty> m = new HashMap<>();
- m.put(CONFIG_ES_ADDRESS, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: http://localhost:9200): ", "http://localhost:9200"));
+ m.put(CONFIG_ES_ADDRESSES, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: localhost:9200): ", "localhost:9200"));
+ m.put(CONFIG_ES_SSL_ENABLED, new MigrationConfigProperty("Should the ElasticSearch TARGET connection be established using SSL (https) protocol ? (yes/no)", null));
m.put(CONFIG_ES_LOGIN, new MigrationConfigProperty("Enter ElasticSearch TARGET login (default: none): ", ""));
m.put(CONFIG_ES_PASSWORD, new MigrationConfigProperty("Enter ElasticSearch TARGET password (default: none): ", ""));
m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ? (yes/no)", null));
@@ -64,47 +60,15 @@ public class MigrationConfig {
configProperties = Collections.unmodifiableMap(m);
}
- Map<String, String> initialConfig = new HashMap<>();
- Map<String, String> computeConfig = new HashMap<>();
+ private Map<String, String> config = new HashMap<>();
@Activate
@Modified
public void modified(Map<String, String> config) {
- initialConfig = config;
- reset();
+ this.config = config;
}
- /**
- * Used reset user choices to initial file system config (useful at the beginning of each new migrate session)
- */
- public void reset() {
- computeConfig.clear();
- computeConfig.putAll(initialConfig);
- }
-
- public String getString(String name, Session session) throws IOException {
- if (computeConfig.containsKey(name)) {
- return computeConfig.get(name);
- }
- if (configProperties.containsKey(name)) {
- MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
- String answer = ConsoleUtils.askUserWithDefaultAnswer(session, migrateConfigProperty.getDescription(), migrateConfigProperty.getDefaultValue());
- computeConfig.put(name, answer);
- return answer;
- }
- return null;
- }
-
- public boolean getBoolean(String name, Session session) throws IOException {
- if (computeConfig.containsKey(name)) {
- return Boolean.parseBoolean(computeConfig.get(name));
- }
- if (configProperties.containsKey(name)) {
- MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
- boolean answer = ConsoleUtils.askUserWithAuthorizedAnswer(session, migrateConfigProperty.getDescription(), Arrays.asList("yes", "no")).equalsIgnoreCase("yes");
- computeConfig.put(name, answer ? "true" : "false");
- return answer;
- }
- return false;
+ protected Map<String, String> getConfig() {
+ return this.config;
}
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfigProperty.java
similarity index 90%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfigProperty.java
index 5006d3fbb..37de28b8c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationConfigProperty.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfigProperty.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
/**
* Just a bean for a configuration property to be used during migration process
@@ -23,7 +23,7 @@ public class MigrationConfigProperty {
String description;
String defaultValue;
- public MigrationConfigProperty(String description, String defaultValue) {
+ protected MigrationConfigProperty(String description, String defaultValue) {
this.description = description;
this.defaultValue = defaultValue;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
new file mode 100644
index 000000000..e86a4e18e
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration.service;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.karaf.shell.api.console.Session;
+import org.jline.reader.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
+import static org.apache.unomi.shell.migration.service.MigrationServiceImpl.MIGRATION_FS_ROOT_FOLDER;
+
+/**
+ * This class is instantiated for each migration process, it contains useful methods to handle the current migration lifecycle.
+ *
+ * This class allow for keeping track of the migration steps by persisting the steps and there state on the FileSystem,
+ * allowing for a migration to be able to restart from a failure in case it happens.
+ *
+ * This class allow also for logging the migration informations depending on the current context:
+ * - if executed in karaf shell using a karaf shell session, then the logging will be done in the shell console
+ * - if executed outside karaf shell using OSGI service direct, then the logging will be done using classical logger systems
+ *
+ * This class allow also to do a best effort on missing configuration information, by prompting questions in the karaf shell
+ * (not supported in case direct OSGI service usage)
+ */
+public class MigrationContext {
+ private static final Logger logger = LoggerFactory.getLogger(MigrationContext.class);
+
+ private static final Path MIGRATION_FS_HISTORY_FILE = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "history.json");
+
+ private enum MigrationStepState {
+ STARTED,
+ COMPLETED
+ }
+
+ protected MigrationContext(Session session, MigrationConfig migrationConfig) {
+ this.session = session;
+ this.migrationConfig = migrationConfig;
+ this.objectMapper = new ObjectMapper();
+ }
+
+ private final Session session;
+ private final MigrationConfig migrationConfig;
+ private final ObjectMapper objectMapper;
+ private CloseableHttpClient httpClient;
+
+ private Map<String, MigrationStepState> history = new HashMap<>();
+ private Map<String, String> userConfig = new HashMap<>();
+
+ /**
+ * Try to recover from a previous run
+ * I case we found an existing history we will ask if we want to recover or if we want to restart from the beginning
+ * (it is also configurable using the conf: recoverFromHistory)
+ */
+ protected void tryRecoverFromHistory() throws IOException {
+ if (Files.exists(MIGRATION_FS_HISTORY_FILE)) {
+ if (getConfigBoolean(MIGRATION_HISTORY_RECOVER)) {
+ history = objectMapper.readValue(MIGRATION_FS_HISTORY_FILE.toFile(), new TypeReference<Map<String, MigrationStepState>>() {});
+ } else {
+ cleanHistory();
+ }
+ }
+ }
+
+ /**
+ * this method allow for migration step execution:
+ * - in case the history already contains the given stepKey as COMPLETED, then the step won't be executed
+ * - in case the history doesn't contain the given stepKey, then the step will be executed
+ * Also this method is keeping track of the history by persisting it on the FileSystem.
+ *
+ * @param stepKey the key of the given step
+ * @param step the step to be performed
+ * @throws IOException
+ */
+ public void performMigrationStep(String stepKey, MigrationStep step) throws Exception {
+ if (step == null || stepKey == null) {
+ throw new IllegalArgumentException("Migration step and/or key cannot be null");
+ }
+
+ // check if step already exists in history:
+ MigrationStepState stepState = history.get(stepKey);
+ if (stepState != MigrationStepState.COMPLETED) {
+ updateHistoryStep(stepKey, MigrationStepState.STARTED);
+ step.execute();
+ updateHistoryStep(stepKey, MigrationStepState.COMPLETED);
+ } else {
+ printMessage("Migration step: " + stepKey + " already completed in previous run");
+ }
+ }
+
+ /**
+ * Clean history from FileSystem
+ * @throws IOException
+ */
+ protected void cleanHistory() throws IOException {
+ Files.deleteIfExists(MIGRATION_FS_HISTORY_FILE);
+ }
+
+ /**
+ * This will ask a question to the user and return the default answer if the user does not answer.
+ *
+ * @param msg String message to ask
+ * @param defaultAnswer String default answer
+ * @return the user's answer
+ * @throws IOException if there was a problem reading input from the console
+ */
+ public String askUserWithDefaultAnswer(String msg, String defaultAnswer) throws IOException {
+ String answer = promptMessageToUser(msg);
+ if (StringUtils.isBlank(answer)) {
+ return defaultAnswer;
+ }
+ return answer;
+ }
+
+ /**
+ * This method allow you to ask a question to the user.
+ * The answer is controlled before being return so the question will be ask until the user enter one the authorized answer
+ *
+ * @param msg String message to ask
+ * @param authorizedAnswer Array of possible answer, all answer must be in lower case
+ * @return the user answer
+ * @throws IOException if there was an error retrieving an answer from the user on the console
+ */
+ public String askUserWithAuthorizedAnswer(String msg, List<String> authorizedAnswer) throws IOException {
+ String answer;
+ do {
+ answer = promptMessageToUser(msg);
+ } while (!authorizedAnswer.contains(answer.toLowerCase()));
+ return answer;
+ }
+
+ /**
+ * This method allow you to prompt a message to the user.
+ * No control is done on the answer provided by the user.
+ *
+ * @param msg String message to prompt
+ * @return the user answer
+ */
+ public String promptMessageToUser(String msg) {
+ if (session == null) {
+ throw new IllegalStateException("Cannot prompt message: " + msg + " to user. " +
+ "(In case you are using the migration tool out of Karaf shell context, please check the migration configuration: org.apache.unomi.migration.cfg)");
+ }
+ LineReader reader = (LineReader) session.get(".jline.reader");
+ return reader.readLine(msg, null);
+ }
+
+ /**
+ * Print a message in the console.
+ * @param msg the message to print out with a newline
+ */
+ public void printMessage(String msg) {
+ if (session == null) {
+ logger.info(msg);
+ } else {
+ PrintStream writer = session.getConsole();
+ writer.println(msg);
+ }
+ }
+
+ /**
+ * Print an exception along with a message in the console.
+ * @param msg the message to print out with a newline
+ * @param t the exception to dump in the shell console after the message
+ */
+ public void printException(String msg, Throwable t) {
+ if (session == null) {
+ logger.error(msg, t);
+ } else {
+ PrintStream writer = session.getConsole();
+ writer.println(msg);
+ t.printStackTrace(writer);
+ }
+ }
+
+ /**
+ * Get config for property name, in case the property doesn't exist on file system config file
+ * Best effort will be made to prompt question in karaf shell to get the needed information
+ *
+ * @param name the name of the property
+ * @return the value of the property
+ * @throws IOException
+ */
+ public String getConfigString(String name) throws IOException {
+ // special handling for esAddress that need to be built
+ if (CONFIG_ES_ADDRESS.equals(name)) {
+ String esAddresses = getConfigString(CONFIG_ES_ADDRESSES);
+ boolean sslEnabled = getConfigBoolean(CONFIG_ES_SSL_ENABLED);
+ return (sslEnabled ? "https://" : "http://") + esAddresses.split(",")[0].trim();
+ }
+
+ if (migrationConfig.getConfig().containsKey(name)) {
+ return migrationConfig.getConfig().get(name);
+ }
+ if (userConfig.containsKey(name)) {
+ return userConfig.get(name);
+ }
+ if (configProperties.containsKey(name)) {
+ MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
+ String answer = askUserWithDefaultAnswer(migrateConfigProperty.getDescription(), migrateConfigProperty.getDefaultValue());
+ userConfig.put(name, answer);
+ return answer;
+ }
+ return null;
+ }
+
+ /**
+ * Get config for property name, in case the property doesn't exist on file system config file
+ * Best effort will be made to prompt question in karaf shell to get the needed information
+ *
+ * @param name the name of the property
+ * @return the value of the property
+ * @throws IOException
+ */
+ public boolean getConfigBoolean(String name) throws IOException {
+ if (migrationConfig.getConfig().containsKey(name)) {
+ return Boolean.parseBoolean(migrationConfig.getConfig().get(name));
+ }
+ if (userConfig.containsKey(name)) {
+ return Boolean.parseBoolean(userConfig.get(name));
+ }
+ if (configProperties.containsKey(name)) {
+ MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
+ boolean answer = askUserWithAuthorizedAnswer(migrateConfigProperty.getDescription(), Arrays.asList("yes", "no")).equalsIgnoreCase("yes");
+ userConfig.put(name, answer ? "true" : "false");
+ return answer;
+ }
+ return false;
+ }
+
+
+ private void updateHistoryStep(String stepKey, MigrationStepState stepState) throws IOException {
+ printMessage("Migration step: " + stepKey + " reach: " + stepState);
+ history.put(stepKey, stepState);
+ objectMapper.writeValue(MIGRATION_FS_HISTORY_FILE.toFile(), history);
+ }
+
+ public CloseableHttpClient getHttpClient() {
+ return httpClient;
+ }
+
+ protected void setHttpClient(CloseableHttpClient httpClient) {
+ this.httpClient = httpClient;
+ }
+
+ /**
+ * A simple migration step to be performed
+ */
+ public interface MigrationStep {
+ /**
+ * Do you migration a safe and unitary way, so that in case this step fail it can be re-executed safely
+ */
+ void execute() throws Exception;
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationScript.java
similarity index 90%
rename from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
rename to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationScript.java
index 15533f056..df3eab2ce 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/MigrationScript.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationScript.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
import groovy.lang.Script;
import org.apache.commons.io.IOUtils;
@@ -48,7 +48,7 @@ public class MigrationScript implements Comparable<MigrationScript> {
private final int priority;
private final String name;
- public MigrationScript(URL scriptURL, Bundle bundle) throws IOException {
+ protected MigrationScript(URL scriptURL, Bundle bundle) throws IOException {
this.bundle = bundle;
this.script = IOUtils.toString(scriptURL);
@@ -65,31 +65,31 @@ public class MigrationScript implements Comparable<MigrationScript> {
}
}
- public Script getCompiledScript() {
+ protected Script getCompiledScript() {
return compiledScript;
}
- public void setCompiledScript(Script compiledScript) {
+ protected void setCompiledScript(Script compiledScript) {
this.compiledScript = compiledScript;
}
- public String getScript() {
+ protected String getScript() {
return script;
}
- public Bundle getBundle() {
+ protected Bundle getBundle() {
return bundle;
}
- public Version getVersion() {
+ protected Version getVersion() {
return version;
}
- public int getPriority() {
+ protected int getPriority() {
return priority;
}
- public String getName() {
+ protected String getName() {
return name;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationServiceImpl.java
similarity index 59%
copy from tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
copy to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationServiceImpl.java
index a21b5d8c3..232940c85 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationServiceImpl.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.unomi.shell.migration.actions;
+package org.apache.unomi.shell.migration.service;
import groovy.lang.GroovyClassLoader;
import groovy.lang.GroovyShell;
@@ -25,16 +25,18 @@ import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.karaf.shell.api.action.Action;
-import org.apache.karaf.shell.api.action.Argument;
-import org.apache.karaf.shell.api.action.Command;
-import org.apache.karaf.shell.api.action.lifecycle.Reference;
-import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.apache.unomi.shell.migration.MigrationService;
import org.apache.unomi.shell.migration.utils.HttpUtils;
-import org.osgi.framework.*;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.Version;
import org.osgi.framework.wiring.BundleWiring;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
import java.io.IOException;
import java.net.URL;
@@ -42,113 +44,121 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.unomi.shell.migration.actions.MigrationConfig.*;
+import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
-@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version. " +
- "It's possible to configure the migration using OSGI configuration file: org.apache.unomi.migration.cfg, if no configuration is provided then questions will be prompted during the migration process.")
-@Service
-public class Migrate implements Action {
+@Component(service = MigrationService.class, immediate = true)
+public class MigrationServiceImpl implements MigrationService {
- protected static final String MIGRATION_FS_ROOT_FOLDER = "migration";
- protected static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
+ public static final String MIGRATION_FS_ROOT_FOLDER = "migration";
+ public static final Path MIGRATION_FS_SCRIPTS_FOLDER = Paths.get(System.getProperty( "karaf.data" ), MIGRATION_FS_ROOT_FOLDER, "scripts");
- @Reference
- Session session;
+ private BundleContext bundleContext;
- @Reference
- BundleContext bundleContext;
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ private MigrationConfig migrationConfig;
- @Reference
- MigrationConfig migrationConfig;
-
- @Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
- private String originVersion;
+ @Activate
+ public void activate(ComponentContext componentContext) {
+ this.bundleContext = componentContext.getBundleContext();
+ }
- @Argument(index = 1, name = "skipConfirmation", description = "Should the confirmation before starting the migration process be skipped ?", valueToShowInHelp = "false")
- private boolean skipConfirmation = false;
+ public void migrateUnomi(String originVersion, boolean skipConfirmation, Session session) throws Exception {
+ System.out.println("Migrating Unomi...");
+ // Wait for config to be loaded by file install, in case of unomi.autoMigrate the OSGI conf may take a few seconds to be loaded correctly.
+ waitForMigrationConfigLoad(60, 1);
- public Object execute() throws Exception {
// Load migration scrips
Set<MigrationScript> scripts = loadOSGIScripts();
scripts.addAll(loadFileSystemScripts());
+ // Create migration context
+ Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
+ MigrationContext context = new MigrationContext(session, migrationConfig);
+ context.tryRecoverFromHistory();
+
+ // no origin version, just print available scripts
if (originVersion == null) {
- displayMigrations(scripts);
- ConsoleUtils.printMessage(session, "Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
- return null;
+ displayMigrations(scripts, context);
+ context.printMessage("Select your migration starting point by specifying the current version (e.g. 1.2.0) or the last script that was already run (e.g. 1.2.1)");
+ return;
}
// Check that there is some migration scripts available from given version
Version fromVersion = new Version(originVersion);
scripts = filterScriptsFromVersion(scripts, fromVersion);
if (scripts.size() == 0) {
- ConsoleUtils.printMessage(session, "No migration scripts available found starting from version: " + originVersion);
- return null;
+ context.printMessage("No migration scripts available found starting from version: " + originVersion);
+ return;
} else {
- ConsoleUtils.printMessage(session, "The following migration scripts starting from version: " + originVersion + " will be executed.");
- displayMigrations(scripts);
+ context.printMessage("The following migration scripts starting from version: " + originVersion + " will be executed.");
+ displayMigrations(scripts, context);
}
// Check for user approval before migrate
- if (!skipConfirmation && ConsoleUtils.askUserWithAuthorizedAnswer(session,
+ if (!skipConfirmation && context.askUserWithAuthorizedAnswer(
"[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ",
Arrays.asList("yes", "no")).equalsIgnoreCase("no")) {
- ConsoleUtils.printMessage(session, "Migration process aborted");
- return null;
+ context.printMessage("Migration process aborted");
+ return;
}
- // reset migration config from previous stored users choices.
- migrationConfig.reset();
- Files.createDirectories(MIGRATION_FS_SCRIPTS_FOLDER);
- MigrationHistory migrationHistory = new MigrationHistory(session, migrationConfig);
- migrationHistory.tryRecover();
-
// Handle credentials
CredentialsProvider credentialsProvider = null;
- String login = migrationConfig.getString(CONFIG_ES_LOGIN, session);
+ String login = context.getConfigString(CONFIG_ES_LOGIN);
if (StringUtils.isNotEmpty(login)) {
credentialsProvider = new BasicCredentialsProvider();
UsernamePasswordCredentials credentials
- = new UsernamePasswordCredentials(login, migrationConfig.getString(CONFIG_ES_PASSWORD, session));
+ = new UsernamePasswordCredentials(login, context.getConfigString(CONFIG_ES_PASSWORD));
credentialsProvider.setCredentials(AuthScope.ANY, credentials);
}
- try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session), credentialsProvider)) {
+ try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(context.getConfigBoolean(CONFIG_TRUST_ALL_CERTIFICATES), credentialsProvider)) {
// Compile scripts
- scripts = parseScripts(scripts, session, httpClient, migrationConfig, migrationHistory);
+ context.setHttpClient(httpClient);
+ scripts = parseScripts(scripts, context);
// Start migration
- ConsoleUtils.printMessage(session, "Starting migration process from version: " + originVersion);
+ context.printMessage("Starting migration process from version: " + originVersion);
for (MigrationScript migrateScript : scripts) {
- ConsoleUtils.printMessage(session, "Starting execution of: " + migrateScript);
+ context.printMessage("Starting execution of: " + migrateScript);
try {
migrateScript.getCompiledScript().run();
} catch (Exception e) {
- ConsoleUtils.printException(session, "Error executing: " + migrateScript, e);
- return null;
+ context.printException("Error executing: " + migrateScript, e);
+ throw e;
}
- ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
+ context.printMessage("Finish execution of: " + migrateScript);
}
// We clean history, migration is successful
- migrationHistory.clean();
+ context.cleanHistory();
}
+ }
- return null;
+ private void waitForMigrationConfigLoad(int maxTry, int secondsToSleep) throws InterruptedException {
+ while (!migrationConfig.getConfig().containsKey("felix.fileinstall.filename")) {
+ maxTry -= 1;
+ if (maxTry == 0) {
+ throw new IllegalStateException("Waited too long for migration config to be available");
+ } else {
+ TimeUnit.SECONDS.sleep(secondsToSleep);
+ }
+ }
}
- private void displayMigrations(Set<MigrationScript> scripts) {
+ private void displayMigrations(Set<MigrationScript> scripts, MigrationContext context) {
Version previousVersion = new Version("0.0.0");
for (MigrationScript migration : scripts) {
if (migration.getVersion().getMajor() > previousVersion.getMajor() || migration.getVersion().getMinor() > previousVersion.getMinor()) {
- ConsoleUtils.printMessage(session, "From " + migration.getVersion().getMajor() + "." + migration.getVersion().getMinor() + ".0:");
+ context.printMessage("From " + migration.getVersion().getMajor() + "." + migration.getVersion().getMinor() + ".0:");
}
- ConsoleUtils.printMessage(session, "- " + migration);
+ context.printMessage("- " + migration);
previousVersion = migration.getVersion();
}
}
@@ -159,7 +169,7 @@ public class Migrate implements Action {
.collect(Collectors.toCollection(TreeSet::new));
}
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
+ private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, MigrationContext context) {
Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
return scripts.stream()
@@ -167,7 +177,7 @@ public class Migrate implements Action {
// fallback on current bundle if the scripts is not provided by OSGI
Bundle scriptBundle = migrateScript.getBundle() != null ? migrateScript.getBundle() : bundleContext.getBundle();
if (!shellsPerBundle.containsKey(scriptBundle.getSymbolicName())) {
- shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, session, httpClient, migrationConfig, migrationHistory));
+ shellsPerBundle.put(scriptBundle.getSymbolicName(), buildShellForBundle(scriptBundle, context));
}
migrateScript.setCompiledScript(shellsPerBundle.get(scriptBundle.getSymbolicName()).parse(migrateScript.getScript()));
})
@@ -212,15 +222,16 @@ public class Migrate implements Action {
return migrationScripts;
}
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, MigrationHistory migrationHistory) {
+ private GroovyShell buildShellForBundle(Bundle bundle, MigrationContext context) {
GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
- groovyShell.setVariable("session", session);
- groovyShell.setVariable("httpClient", httpClient);
- groovyShell.setVariable("migrationConfig", migrationConfig);
- groovyShell.setVariable("migrationHistory", migrationHistory);
+ groovyShell.setVariable("migrationContext", context);
groovyShell.setVariable("bundleContext", bundle.getBundleContext());
return groovyShell;
}
+
+ public void setMigrationConfig(MigrationConfig migrationConfig) {
+ this.migrationConfig = migrationConfig;
+ }
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
deleted file mode 100644
index b56a7568c..000000000
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.shell.migration.utils;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.karaf.shell.api.console.Session;
-import org.jline.reader.LineReader;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-
-/**
- * @author dgaillard
- */
-public class ConsoleUtils {
-
- /**
- * This will ask a question to the user and return the default answer if the user does not answer.
- *
- * @param session the shell's session
- * @param msg String message to ask
- * @param defaultAnswer String default answer
- * @return the user's answer
- * @throws IOException if there was a problem reading input from the console
- */
- public static String askUserWithDefaultAnswer(Session session, String msg, String defaultAnswer) throws IOException {
- String answer = promptMessageToUser(session, msg);
- if (StringUtils.isBlank(answer)) {
- return defaultAnswer;
- }
- return answer;
- }
-
- /**
- * This method allow you to ask a question to the user.
- * The answer is controlled before being return so the question will be ask until the user enter one the authorized answer
- *
- * @param session the shell's session
- * @param msg String message to ask
- * @param authorizedAnswer Array of possible answer, all answer must be in lower case
- * @return the user answer
- * @throws IOException if there was an error retrieving an answer from the user on the console
- */
- public static String askUserWithAuthorizedAnswer(Session session, String msg, List<String> authorizedAnswer) throws IOException {
- String answer;
- do {
- answer = promptMessageToUser(session,msg);
- } while (!authorizedAnswer.contains(answer.toLowerCase()));
- return answer;
- }
-
- /**
- * This method allow you to prompt a message to the user.
- * No control is done on the answer provided by the user.
- *
- * @param session the shell's session
- * @param msg String message to prompt
- * @return the user answer
- */
- public static String promptMessageToUser(Session session, String msg) {
- LineReader reader = (LineReader) session.get(".jline.reader");
- return reader.readLine(msg, null);
- }
-
- /**
- * Print a message in the console.
- * @param session the shell's session
- * @param msg the message to print out with a newline
- */
- public static void printMessage(Session session, String msg) {
- PrintStream writer = session.getConsole();
- writer.println(msg);
- }
-
- /**
- * Print an exception along with a message in the console.
- * @param session the shell's session
- * @param msg the message to print out with a newline
- * @param t the exception to dump in the shell console after the message
- */
- public static void printException(Session session, String msg, Throwable t) {
- PrintStream writer = session.getConsole();
- writer.println(msg);
- t.printStackTrace(writer);
- }
-}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 9840b0184..d44be389c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -23,7 +23,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
-import org.apache.unomi.shell.migration.actions.MigrationHistory;
+import org.apache.unomi.shell.migration.service.MigrationContext;
import org.json.JSONArray;
import org.json.JSONObject;
import org.osgi.framework.Bundle;
@@ -142,7 +142,7 @@ public class MigrationUtils {
}
public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
- String newIndexSettings, String painlessScript, MigrationHistory history) throws Exception {
+ String newIndexSettings, String painlessScript, MigrationContext migrationContext) throws Exception {
if (indexName.endsWith("-cloned")) {
// We should never reIndex a clone ...
return;
@@ -156,7 +156,7 @@ public class MigrationUtils {
String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json");
- history.performMigrationStep("Reindex step for: " + indexName + " (clone creation)", () -> {
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (clone creation)", () -> {
// Delete clone in case it already exists, could be incomplete from a previous reindex attempt, so better create a fresh one.
if (indexExists(httpClient, esAddress, indexNameCloned)) {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
@@ -167,7 +167,7 @@ public class MigrationUtils {
HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_clone/" + indexNameCloned, null, null);
});
- history.performMigrationStep("Reindex step for: " + indexName + " (recreate the index and perform the re-indexation)", () -> {
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (recreate the index and perform the re-indexation)", () -> {
// Delete original index if it still exists
if (indexExists(httpClient, esAddress, indexName)) {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null);
@@ -178,7 +178,7 @@ public class MigrationUtils {
HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
});
- history.performMigrationStep("Reindex step for: " + indexName + " (delete clone)", () -> {
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (delete clone)", () -> {
// Delete original index if it still exists
if (indexExists(httpClient, esAddress, indexNameCloned)) {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
index 1619d275c..45c1e0279 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
@@ -17,10 +17,16 @@
package org.apache.unomi.shell.services.internal;
import org.apache.commons.lang3.StringUtils;
+import org.apache.unomi.shell.migration.MigrationService;
import org.apache.unomi.shell.services.UnomiManagementService;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
import org.osgi.framework.BundleException;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.ReferenceCardinality;
import java.util.ArrayList;
import java.util.Collections;
@@ -29,14 +35,26 @@ import java.util.List;
/**
* @author dgaillard
*/
+@Component(service = UnomiManagementService.class, immediate = true)
public class UnomiManagementServiceImpl implements UnomiManagementService {
private BundleContext bundleContext;
- private List<String> bundleSymbolicNames;
+
+ @Reference(cardinality = ReferenceCardinality.MANDATORY)
+ private MigrationService migrationService;
+
+ private final List<String> bundleSymbolicNames = new ArrayList<>();
private List<String> reversedBundleSymbolicNames;
- public void init() throws BundleException {
+ @Activate
+ public void init(ComponentContext componentContext) throws Exception {
+ this.bundleContext = componentContext.getBundleContext();
initReversedBundleSymbolicNames();
+
+ if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoMigrate"))) {
+ migrationService.migrateUnomi(bundleContext.getProperty("unomi.autoMigrate"), true, null);
+ }
+
if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) && bundleContext.getProperty("unomi.autoStart")
.equals("true")) {
startUnomi();
@@ -71,20 +89,43 @@ public class UnomiManagementServiceImpl implements UnomiManagementService {
}
}
- public void setBundleContext(BundleContext bundleContext) {
- this.bundleContext = bundleContext;
- }
-
- public void setBundleSymbolicNames(List<String> bundleSymbolicNames) {
- this.bundleSymbolicNames = bundleSymbolicNames;
- }
-
public void initReversedBundleSymbolicNames() {
+ bundleSymbolicNames.clear();
+ bundleSymbolicNames.add("org.apache.unomi.lifecycle-watcher");
+ bundleSymbolicNames.add("org.apache.unomi.api");
+ bundleSymbolicNames.add("org.apache.unomi.common");
+ bundleSymbolicNames.add("org.apache.unomi.scripting");
+ bundleSymbolicNames.add("org.apache.unomi.metrics");
+ bundleSymbolicNames.add("org.apache.unomi.persistence-spi");
+ bundleSymbolicNames.add("org.apache.unomi.persistence-elasticsearch-core");
+ bundleSymbolicNames.add("org.apache.unomi.services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-rest");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-geonames-services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-geonames-rest");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-privacy-extension-services");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-privacy-extension-rest");
+ bundleSymbolicNames.add("org.apache.unomi.json-schema-services");
+ bundleSymbolicNames.add("org.apache.unomi.json-schema-rest");
+ bundleSymbolicNames.add("org.apache.unomi.rest");
+ bundleSymbolicNames.add("org.apache.unomi.wab");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-base");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-request");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-mail");
+ bundleSymbolicNames.add("org.apache.unomi.plugins-optimization-test");
+ bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-actions");
+ bundleSymbolicNames.add("org.apache.unomi.router-api");
+ bundleSymbolicNames.add("org.apache.unomi.router-core");
+ bundleSymbolicNames.add("org.apache.unomi.router-service");
+ bundleSymbolicNames.add("org.apache.unomi.router-rest");
+ bundleSymbolicNames.add("org.apache.unomi.shell-dev-commands");
+ bundleSymbolicNames.add("org.apache.unomi.groovy-actions-services");
+ bundleSymbolicNames.add("org.apache.unomi.groovy-actions-rest");
+
if (reversedBundleSymbolicNames == null || reversedBundleSymbolicNames.isEmpty()) {
this.reversedBundleSymbolicNames = new ArrayList<>();
reversedBundleSymbolicNames.addAll(bundleSymbolicNames);
Collections.reverse(reversedBundleSymbolicNames);
}
}
-
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy
index 152f432a1..45f7b6fd5 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.1-00-migrateTags.groovy
@@ -17,4 +17,4 @@ import org.apache.unomi.shell.migration.impl.MigrationTo121
* limitations under the License.
*/
-new MigrationTo121().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
+new MigrationTo121().execute(migrationContext, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy
index 99fbd8f66..c9530a646 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.2.2-00-deleteOldIndexTemplate.groovy
@@ -17,4 +17,4 @@ import org.apache.unomi.shell.migration.impl.MigrationTo122
* limitations under the License.
*/
-new MigrationTo122().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
+new MigrationTo122().execute(migrationContext, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy
index 4fbf01423..7151b2f3b 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-1.5.0-00-elasticSearch7.4.groovy
@@ -17,4 +17,4 @@ import org.apache.unomi.shell.migration.impl.MigrationTo150
* limitations under the License.
*/
-new MigrationTo150().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
+new MigrationTo150().execute(migrationContext, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index 9dc340405..dcc2a0511 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -1,7 +1,5 @@
import groovy.json.JsonSlurper
-import org.apache.http.impl.client.CloseableHttpClient
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -25,33 +23,32 @@ import java.time.Instant
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-CloseableHttpClient client = httpClient
+MigrationContext context = migrationContext
Instant migrationTime = Instant.now()
def jsonSlurper = new JsonSlurper()
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String aliasIndex = indexPrefix + "-profilealias"
String profileIndex = indexPrefix + "-profile"
-history.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
- if (!MigrationUtils.indexExists(client, esAddress, aliasIndex)) {
+context.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, aliasIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, profileIndex, mapping)
- HttpUtils.executePutRequest(client, esAddress + "/" + aliasIndex, newIndexSettings, null)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, profileIndex, mapping)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + aliasIndex, newIndexSettings, null)
}
})
-history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles", () -> {
+context.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles", () -> {
String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
String profileMergedSearchRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_search.json")
- MigrationUtils.scrollQuery(client, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", hits -> {
+ MigrationUtils.scrollQuery(context.getHttpClient(), esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", hits -> {
// create aliases for those merged profiles and delete them.
def jsonHits = jsonSlurper.parseText(hits)
- ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " existing profiles merged")
+ context.printMessage("Detected: " + jsonHits.size() + " existing profiles merged")
final StringBuilder bulkSaveRequest = new StringBuilder()
jsonHits.each {
@@ -63,14 +60,14 @@ history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles"
def aliasAlreadyExists = false
try {
- def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + profileIndex + "/_doc/" + masterProfileId, null))
+ def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(context.getHttpClient(), esAddress + "/" + profileIndex + "/_doc/" + masterProfileId, null))
masterProfileExists = masterProfile.found
} catch (HttpRequestException e) {
// can happen in case response code > 400 due to item not exist in ElasticSearch
}
try {
- def existingAlias = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + aliasIndex + "/_doc/" + mergedProfileId, null));
+ def existingAlias = jsonSlurper.parseText(HttpUtils.executeGetRequest(context.getHttpClient(), esAddress + "/" + aliasIndex + "/_doc/" + mergedProfileId, null));
aliasAlreadyExists = existingAlias.found
} catch (HttpRequestException e) {
// can happen in case of response code > 400 due to item not exist in ElasticSearch
@@ -86,12 +83,12 @@ history.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles"
}
if (bulkSaveRequest.length() > 0) {
- HttpUtils.executePostRequest(client, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
})
})
-history.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
+context.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
- HttpUtils.executePostRequest(client, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 273389cff..9b38b81f6 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -1,7 +1,5 @@
import groovy.json.JsonSlurper
-import org.apache.http.impl.client.CloseableHttpClient
-import org.apache.unomi.shell.migration.actions.MigrationHistory
-import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -23,28 +21,27 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-CloseableHttpClient client = httpClient
+MigrationContext context = migrationContext
def jsonSlurper = new JsonSlurper()
String searchScopesRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/scope_search.json")
String saveScopeRequestBulk = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scope_save_bulk.ndjson")
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String scopeIndex = indexPrefix + "-scope"
-history.performMigrationStep("2.0.0-create-scope-index", () -> {
- if (!MigrationUtils.indexExists(client, esAddress, scopeIndex)) {
+context.performMigrationStep("2.0.0-create-scope-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(client, esAddress, baseRequest, indexPrefix + "-profile", mapping)
- HttpUtils.executePutRequest(client, esAddress + "/" + scopeIndex, newIndexSettings, null)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, indexPrefix + "-profile", mapping)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null)
}
})
-history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
+context.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
// search existing scopes from event
- def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(client, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
- ConsoleUtils.printMessage(session, "Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
+ def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
+ context.printMessage("Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
// create scopes
def buckets = searchResponse.aggregations.scopes.buckets
@@ -58,7 +55,7 @@ history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
// check that the scope doesn't already exists
def scopeAlreadyExists = false
try {
- def existingScope = jsonSlurper.parseText(HttpUtils.executeGetRequest(client, esAddress + "/" + scopeIndex + "/_doc/" + bucket.key, null));
+ def existingScope = jsonSlurper.parseText(HttpUtils.executeGetRequest(context.getHttpClient(), esAddress + "/" + scopeIndex + "/_doc/" + bucket.key, null));
scopeAlreadyExists = existingScope.found
} catch (HttpRequestException e) {
// can happen in case response code > 400 due to item not exist in ElasticSearch
@@ -72,7 +69,7 @@ history.performMigrationStep("2.0.0-create-scopes-from-existing-events", () -> {
}
if (bulkSaveRequest.length() > 0) {
- HttpUtils.executePostRequest(client, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
}
}
})
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 4d1df4791..211261a0f 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -1,4 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -18,9 +18,9 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"];
@@ -28,6 +28,6 @@ indicesToReindex.each { indexToReindex ->
String mappingFileName = "${indexToReindex}.json"
String realIndexName = "${indexPrefix}-${indexToReindex.toLowerCase()}"
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, mappingFileName)
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, realIndexName, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, realIndexName, newIndexSettings, null, history)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, realIndexName, mapping)
+ MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, realIndexName, newIndexSettings, null, context)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index b0bc12d6e..52e532bf4 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -1,4 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.MigrationUtils
/*
@@ -18,12 +18,12 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, indexPrefix + "-profile", mapping)
-MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), history)
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, indexPrefix + "-profile", mapping)
+MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, indexPrefix + "-profile",
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), context)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index 492c7ae1a..a98818118 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -1,4 +1,4 @@
-import org.apache.unomi.shell.migration.actions.MigrationHistory
+import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpUtils
import org.apache.unomi.shell.migration.utils.MigrationUtils
@@ -19,21 +19,21 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-MigrationHistory history = migrationHistory
-String esAddress = migrationConfig.getString("esAddress", session)
-String indexPrefix = migrationConfig.getString("indexPrefix", session)
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
-history.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
+context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
- HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+ HttpUtils.executePostRequest(context.getHttpClient(), "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
})
// Reindex the rest of the events
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
-Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, "${indexPrefix}-event-")
+Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-")
eventIndices.each { eventIndex ->
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseSettings, eventIndex, mapping)
- MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, history)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, eventIndex, mapping)
+ MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context)
}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/tools/shell-commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml
deleted file mode 100644
index 61f70c335..000000000
--- a/tools/shell-commands/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<blueprint xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
- xsi:schemaLocation="http://www.osgi.org/xmlns/blueprint/v1.0.0 http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd">
-
- <bean id="unomiManagementServiceImpl" class="org.apache.unomi.shell.services.internal.UnomiManagementServiceImpl" init-method="init">
- <property name="bundleSymbolicNames">
- <list>
- <value>org.apache.unomi.lifecycle-watcher</value>
- <value>org.apache.unomi.api</value>
- <value>org.apache.unomi.common</value>
- <value>org.apache.unomi.scripting</value>
- <value>org.apache.unomi.metrics</value>
- <value>org.apache.unomi.persistence-spi</value>
- <value>org.apache.unomi.persistence-elasticsearch-core</value>
- <value>org.apache.unomi.services</value>
- <value>org.apache.unomi.cxs-lists-extension-services</value>
- <value>org.apache.unomi.cxs-lists-extension-rest</value>
- <value>org.apache.unomi.cxs-geonames-services</value>
- <value>org.apache.unomi.cxs-geonames-rest</value>
- <value>org.apache.unomi.cxs-privacy-extension-services</value>
- <value>org.apache.unomi.cxs-privacy-extension-rest</value>
- <value>org.apache.unomi.json-schema-services</value>
- <value>org.apache.unomi.json-schema-rest</value>
- <value>org.apache.unomi.rest</value>
- <value>org.apache.unomi.wab</value>
- <value>org.apache.unomi.plugins-base</value>
- <value>org.apache.unomi.plugins-request</value>
- <value>org.apache.unomi.plugins-mail</value>
- <value>org.apache.unomi.plugins-optimization-test</value>
- <value>org.apache.unomi.cxs-lists-extension-actions</value>
- <value>org.apache.unomi.router-api</value>
- <value>org.apache.unomi.router-core</value>
- <value>org.apache.unomi.router-service</value>
- <value>org.apache.unomi.router-rest</value>
- <value>org.apache.unomi.shell-dev-commands</value>
- <value>org.apache.unomi.groovy-actions-services</value>
- <value>org.apache.unomi.groovy-actions-rest</value>
- </list>
- </property>
- <property name="bundleContext" ref="blueprintBundleContext"/>
- </bean>
- <service id="unomiManagementService" ref="unomiManagementServiceImpl"
- interface="org.apache.unomi.shell.services.UnomiManagementService"/>
-</blueprint>
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index b5dda03ac..5749e4cf9 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -18,12 +18,13 @@
# Migration config used for silent migration
# Various configuration related to ElasticSearch communication to be able to connect and migrate the data
-# esAddress = http://localhost:9200
-# esLogin = elastic
-# esPassword = password
-# httpClient.trustAllCertificates = true
-# indexPrefix = context
+esAddresses = ${org.apache.unomi.elasticsearch.addresses:-localhost:9200}
+esSSLEnabled = ${org.apache.unomi.elasticsearch.sslEnable:-false}
+esLogin = ${org.apache.unomi.elasticsearch.username:-}
+esPassword = ${org.apache.unomi.elasticsearch.password:-}
+httpClient.trustAllCertificates = ${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false}
+indexPrefix = ${org.apache.unomi.elasticsearch.index.prefix:-context}
# Should the migration try to recover from a previous run ?
# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)
-# recoverFromHistory = true
\ No newline at end of file
+recoverFromHistory = ${org.apache.unomi.migration.recoverFromHistory:-true}
\ No newline at end of file