You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/07/26 09:31:07 UTC
[unomi] branch migrationTests updated: UNOMI-633: implement migration configuration handling and silent migration option
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch migrationTests
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/migrationTests by this push:
new 096c361fd UNOMI-633: implement migration configuration handling and silent migration option
096c361fd is described below
commit 096c361fdb79d10f74b6ae4169fc1a9a1ad4f970
Author: Kevan <ke...@jahia.com>
AuthorDate: Tue Jul 26 11:30:56 2022 +0200
UNOMI-633: implement migration configuration handling and silent migration option
---
kar/src/main/feature/feature.xml | 1 +
tools/shell-commands/pom.xml | 24 +++++
.../apache/unomi/shell/migration/Migration.java | 2 +-
.../unomi/shell/migration/MigrationConfig.java | 104 +++++++++++++++++++++
.../shell/migration/MigrationConfigProperty.java} | 26 +++++-
.../unomi/shell/migration/actions/Migrate.java | 30 +++---
.../unomi/shell/migration/impl/MigrationTo121.java | 5 +-
.../unomi/shell/migration/impl/MigrationTo122.java | 5 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 7 +-
.../unomi/shell/migration/impl/MigrationTo200.java | 35 +++----
.../migrate-2.0.0-01-segmentReindex.groovy | 8 +-
.../migrate-2.0.0-02-scoringPlanReindex.groovy | 8 +-
.../migrate-2.0.0-03-profileReindex.groovy | 7 +-
.../migrate-2.0.0-04-eventsReindex.groovy | 4 +-
.../main/resources/org.apache.unomi.migration.cfg | 26 ++++++
15 files changed, 240 insertions(+), 52 deletions(-)
diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml
index b0c8db5e5..541314a8c 100644
--- a/kar/src/main/feature/feature.xml
+++ b/kar/src/main/feature/feature.xml
@@ -91,6 +91,7 @@
<bundle start-level="85" start="false">mvn:org.apache.unomi/cxs-lists-extension-actions/${project.version}</bundle>
<bundle start-level="85" start="false">mvn:org.apache.unomi/shell-dev-commands/${project.version}</bundle>
+ <configfile finalname="/etc/org.apache.unomi.migration.cfg">mvn:org.apache.unomi/shell-commands/${project.version}/cfg/migration</configfile>
<bundle start-level="99">mvn:org.apache.unomi/shell-commands/${project.version}</bundle>
</feature>
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 57ae9dc3c..e3e863d0a 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -104,6 +104,30 @@
</instructions>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>attach-artifacts</id>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>
+ src/main/resources/org.apache.unomi.migration.cfg
+ </file>
+ <type>cfg</type>
+ <classifier>migration</classifier>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
index 8a64882ce..c7d9ece8c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/Migration.java
@@ -37,5 +37,5 @@ public interface Migration {
* @throws IOException if there was an error while executing the migration
* @deprecated do groovy script for implementing new migrations
*/
- void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException;
+ void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException;
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
new file mode 100644
index 000000000..2e6b99587
--- /dev/null
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.shell.migration;
+
+import org.apache.karaf.shell.api.console.Session;
+import org.apache.unomi.shell.migration.utils.ConsoleUtils;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Modified;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Service uses to aggregate different configuration needed by the migrations
+ * Source of config:
+ * - file system in OSGI config file: org.apache.unomi.migration.cfg
+ * - user interactions in the console during the migration process
+ */
+@Component(immediate = true, service = MigrationConfig.class, configurationPid = {"org.apache.unomi.migration"})
+public class MigrationConfig {
+
+ public static final String CONFIG_ES_ADDRESS = "esAddress";
+ public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
+ public static final String INDEX_PREFIX = "indexPrefix";
+ public static final String NUMBER_OF_SHARDS = "number_of_shards";
+ public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
+ public static final String TOTAL_FIELDS_LIMIT = "mapping.total_fields.limit";
+ public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
+
+ private static final Map<String, MigrationConfigProperty> configProperties;
+ static {
+ Map<String, MigrationConfigProperty> m = new HashMap<>();
+ m.put(CONFIG_ES_ADDRESS, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: http://localhost:9200): ", "http://localhost:9200"));
+ m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ?", null));
+ m.put(INDEX_PREFIX, new MigrationConfigProperty("Enter ElasticSearch Unomi indices prefix (default: context): ", "context"));
+ m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 3): ", "3"));
+ m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
+ m.put(TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
+ m.put(MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
+ configProperties = Collections.unmodifiableMap(m);
+ }
+
+ Map<String, String> initialConfig = new HashMap<>();
+ Map<String, String> computeConfig = new HashMap<>();
+
+ @Activate
+ @Modified
+ public void modified(Map<String, String> config) {
+ initialConfig = config;
+ reset();
+ }
+
+ /**
+ * Used reset user choices to initial file system config (useful at the beginning of each new migrate session)
+ */
+ public void reset() {
+ computeConfig.clear();
+ computeConfig.putAll(initialConfig);
+ }
+
+ public String getString(String name, Session session) throws IOException {
+ if (computeConfig.containsKey(name)) {
+ return computeConfig.get(name);
+ }
+ if (configProperties.containsKey(name)) {
+ MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
+ String answer = ConsoleUtils.askUserWithDefaultAnswer(session, migrateConfigProperty.getDescription(), migrateConfigProperty.getDefaultValue());
+ computeConfig.put(name, answer);
+ return answer;
+ }
+ return null;
+ }
+
+ public boolean getBoolean(String name, Session session) throws IOException {
+ if (computeConfig.containsKey(name)) {
+ return Boolean.parseBoolean(computeConfig.get(name));
+ }
+ if (configProperties.containsKey(name)) {
+ MigrationConfigProperty migrateConfigProperty = configProperties.get(name);
+ boolean answer = ConsoleUtils.askUserWithAuthorizedAnswer(session, migrateConfigProperty.getDescription(), Arrays.asList("yes", "no")).equalsIgnoreCase("yes");
+ computeConfig.put(name, answer ? "true" : "false");
+ return answer;
+ }
+ return false;
+ }
+}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
similarity index 60%
copy from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
copy to tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
index 83ab51233..58f4bee75 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfigProperty.java
@@ -1,5 +1,3 @@
-import org.apache.unomi.shell.migration.utils.MigrationUtils
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,7 +14,25 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.unomi.shell.migration;
+
+/**
+ * Just a bean for a configuration property to be used during migration process
+ */
+public class MigrationConfigProperty {
+ String description;
+ String defaultValue;
+
+ public MigrationConfigProperty(String description, String defaultValue) {
+ this.description = description;
+ this.defaultValue = defaultValue;
+ }
+
+ public String getDescription() {
+ return description;
+ }
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/profile_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"))
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index 80778ec2b..8a3f69bf8 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -26,6 +26,7 @@ import org.apache.karaf.shell.api.action.Command;
import org.apache.karaf.shell.api.action.lifecycle.Reference;
import org.apache.karaf.shell.api.action.lifecycle.Service;
import org.apache.karaf.shell.api.console.Session;
+import org.apache.unomi.shell.migration.MigrationConfig;
import org.apache.unomi.shell.migration.MigrationScript;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpUtils;
@@ -41,12 +42,12 @@ import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.apache.unomi.shell.migration.MigrationConfig.CONFIG_TRUST_ALL_CERTIFICATES;
+
@Command(scope = "unomi", name = "migrate", description = "This will Migrate your data in ES to be compliant with current version")
@Service
public class Migrate implements Action {
- public static final String CONFIG_ES_ADDRESS = "esAddress";
- public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
- public static final String INDEX_PREFIX = "indexPrefix";
+
@Reference
Session session;
@@ -54,9 +55,15 @@ public class Migrate implements Action {
@Reference
BundleContext bundleContext;
+ @Reference
+ MigrationConfig migrationConfig;
+
@Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
private String originVersion;
+ @Argument(index = 1, name = "silent", description = "Should the migration process be silent ? (default: false)", valueToShowInHelp = "true")
+ private boolean silent = false;
+
public Object execute() throws Exception {
// Load migration scrips
Set<MigrationScript> scripts = loadOSGIScripts();
@@ -80,20 +87,17 @@ public class Migrate implements Action {
}
// Check for user approval before migrate
- if (ConsoleUtils.askUserWithAuthorizedAnswer(session,
+ if (!silent && ConsoleUtils.askUserWithAuthorizedAnswer(session,
"[WARNING] You are about to execute a migration, this a very sensitive operation, are you sure? (yes/no): ",
Arrays.asList("yes", "no")).equalsIgnoreCase("no")) {
ConsoleUtils.printMessage(session, "Migration process aborted");
return null;
}
- // Build conf
- Map<String, Object> migrationConfig = new HashMap<>();
- migrationConfig.put(CONFIG_ES_ADDRESS, ConsoleUtils.askUserWithDefaultAnswer(session, "Enter ElasticSearch 7 TARGET address (default = http://localhost:9200): ", "http://localhost:9200"));
- migrationConfig.put(CONFIG_TRUST_ALL_CERTIFICATES, ConsoleUtils.askUserWithAuthorizedAnswer(session,"We need to initialize a HttpClient, do we need to trust all certificates? (yes/no): ", Arrays.asList("yes", "no")).equalsIgnoreCase("yes"));
- migrationConfig.put(INDEX_PREFIX, ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context"));
-
- try (CloseableHttpClient httpClient = HttpUtils.initHttpClient((Boolean) migrationConfig.get(CONFIG_TRUST_ALL_CERTIFICATES))) {
+ // reset migration config from previous stored users choices.
+ migrationConfig.reset();
+
+ try (CloseableHttpClient httpClient = HttpUtils.initHttpClient(migrationConfig.getBoolean(CONFIG_TRUST_ALL_CERTIFICATES, session))) {
// Compile scripts
scripts = parseScripts(scripts, session, httpClient, migrationConfig);
@@ -133,7 +137,7 @@ public class Migrate implements Action {
.collect(Collectors.toCollection(TreeSet::new));
}
- private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig) {
+ private Set<MigrationScript> parseScripts(Set<MigrationScript> scripts, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig) {
Map<String, GroovyShell> shellsPerBundle = new HashMap<>();
return scripts.stream()
@@ -187,7 +191,7 @@ public class Migrate implements Action {
return migrationScripts;
}
- private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig) {
+ private GroovyShell buildShellForBundle(Bundle bundle, Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig) {
GroovyClassLoader groovyLoader = new GroovyClassLoader(bundle.adapt(BundleWiring.class).getClassLoader());
GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine((URL[]) null, groovyLoader);
GroovyShell groovyShell = new GroovyShell(groovyScriptEngine.getGroovyClassLoader());
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
index 06183296f..072e97035 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo121.java
@@ -20,6 +20,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
+import org.apache.unomi.shell.migration.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.MigrationUtils;
import org.json.JSONArray;
@@ -41,10 +42,10 @@ public class MigrationTo121 implements Migration {
private List propsTaggedAsPersonalIdentifier = Arrays.asList("firstName", "lastName", "email", "phoneNumber", "address", "facebookId", "googleId", "linkedInId", "twitterId");
@Override
- public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException {
+ public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
this.httpClient = httpClient;
this.session = session;
- this.esAddress = (String) migrationConfig.get("esAddress");
+ this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
migrateTags();
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
index b371f8a0d..5ad9c28ac 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo122.java
@@ -19,6 +19,7 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
+import org.apache.unomi.shell.migration.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpRequestException;
import org.apache.unomi.shell.migration.utils.HttpUtils;
@@ -34,10 +35,10 @@ public class MigrationTo122 implements Migration {
private String esAddress;
@Override
- public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException {
+ public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
this.httpClient = httpClient;
this.session = session;
- this.esAddress = (String) migrationConfig.get("esAddress");
+ this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
deleteOldIndexTemplate();
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index 4eb84c821..074a7fabf 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -19,6 +19,7 @@ package org.apache.unomi.shell.migration.impl;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
+import org.apache.unomi.shell.migration.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.HttpUtils;
import org.json.JSONArray;
@@ -37,10 +38,10 @@ public class MigrationTo150 implements Migration {
public static final String INDEX_DATE_PREFIX = "date-";
@Override
- public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException {
- String esAddress = (String) migrationConfig.get("esAddress");
+ public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
+ String esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
String es5Address = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
- String sourceIndexPrefix = (String) migrationConfig.get("indexPrefix");
+ String sourceIndexPrefix = migrationConfig.getString(MigrationConfig.INDEX_PREFIX, session);
String destIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "TARGET index prefix (default: context) : ", "context");
int numberOfShards = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of shards for TARGET (default: 5) : ", "5"));
int numberOfReplicas = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of replicas for TARGET (default: 1) : ", "1"));
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index 3e8967ed0..2dcb74bb9 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -27,6 +27,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.karaf.shell.api.console.Session;
import org.apache.unomi.shell.migration.Migration;
+import org.apache.unomi.shell.migration.MigrationConfig;
import org.apache.unomi.shell.migration.utils.ConsoleUtils;
import org.apache.unomi.shell.migration.utils.MigrationUtils;
import org.json.JSONArray;
@@ -48,26 +49,30 @@ public class MigrationTo200 implements Migration {
private CloseableHttpClient httpClient;
private Session session;
private String esAddress;
+ private String indexPrefix;
private BundleContext bundleContext;
+ private MigrationConfig migrationConfig;
@Override
- public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException {
+ public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
this.httpClient = httpClient;
this.session = session;
- this.esAddress = (String) migrationConfig.get("esAddress");
+ this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
+ this.indexPrefix = migrationConfig.getString(MigrationConfig.INDEX_PREFIX, session);
this.bundleContext = bundleContext;
+ this.migrationConfig = migrationConfig;
- doExecute((String) migrationConfig.get("indexPrefix"));
+ doExecute();
}
- private void doExecute(String indexPrefix) throws IOException {
+ private void doExecute() throws IOException {
Set<String> indexes = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, indexPrefix + "-event-");
- createScopeMapping(indexPrefix);
- createScopes(getSetOfScopes(indexes), indexPrefix);
+ createScopeMapping();
+ createScopes(getSetOfScopes(indexes));
createProfileAliasDocumentsFromProfile();
}
- private boolean scopeIndexNotExists(String indexPrefix) throws IOException {
+ private boolean scopeIndexNotExists() throws IOException {
final HttpGet httpGet = new HttpGet(esAddress + "/" + indexPrefix + "-scope");
httpGet.addHeader("Accept", "application/json");
@@ -78,17 +83,15 @@ public class MigrationTo200 implements Migration {
}
}
- private void createScopeMapping(String indexPrefix) throws IOException {
+ private void createScopeMapping() throws IOException {
- if (scopeIndexNotExists(indexPrefix)) {
+ if (scopeIndexNotExists()) {
System.out.println("Creation for index = \"" + indexPrefix + "-scope\" starting.");
System.out.println("Specify the following parameters:");
- String numberOfShards = ConsoleUtils.askUserWithDefaultAnswer(session, "number_of_shards: (default: 3)", "3");
- String numberOfReplicas = ConsoleUtils.askUserWithDefaultAnswer(session, "number_of_replicas: (default: 0)", "0");
- String mappingTotalFieldsLimit = ConsoleUtils
- .askUserWithDefaultAnswer(session, "mapping.total_fields.limit: (default: 1000)", "1000");
- String maxDocValueFieldsSearch = ConsoleUtils
- .askUserWithDefaultAnswer(session, "max_docvalue_fields_search: (default: 1000)", "1000");
+ String numberOfShards = migrationConfig.getString(MigrationConfig.NUMBER_OF_SHARDS, session);
+ String numberOfReplicas = migrationConfig.getString(MigrationConfig.NUMBER_OF_REPLICAS, session);
+ String mappingTotalFieldsLimit = migrationConfig.getString(MigrationConfig.TOTAL_FIELDS_LIMIT, session);
+ String maxDocValueFieldsSearch = migrationConfig.getString(MigrationConfig.MAX_DOC_VALUE_FIELDS_SEARCH, session);
final HttpPut httpPost = new HttpPut(esAddress + "/" + indexPrefix + "-scope");
@@ -116,7 +119,7 @@ public class MigrationTo200 implements Migration {
}
- private void createScopes(Set<String> scopes, String indexPrefix) throws IOException {
+ private void createScopes(Set<String> scopes) throws IOException {
final StringBuilder body = new StringBuilder();
String saveScopeBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/bulkSaveScope.ndjson");
scopes.forEach(scope -> body.append(saveScopeBody.replace("$scope", scope)));
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
index 96048e37f..e653abbbc 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
@@ -17,6 +17,8 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment",
- newIndexSettings, null)
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json")
+MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-segment", newIndexSettings, null)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
index 46e898117..df41aa62d 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
@@ -17,6 +17,8 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scoring_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-scoring",
- newIndexSettings, null)
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scoring_index.json")
+MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-scoring", newIndexSettings, null)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
index 83ab51233..f82794ec2 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
@@ -17,6 +17,9 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/profile_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-profile",
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/profile_index.json")
+MigrationUtils.reIndex(httpClient, bundleContext, esAddress, indexPrefix + "-profile",
newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"))
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
index 10c64646f..4f2751db7 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
@@ -18,8 +18,8 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String esAddress = migrationConfig.get("esAddress")
-String indexPrefix = migrationConfig.get("indexPrefix")
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
// Remove all internal events that are no more persisted
String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
new file mode 100644
index 000000000..8c1a5dc0f
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Migration config used for silent migration
+
+# esAddress = http://localhost:9200
+# httpClient.trustAllCertificates = true
+# indexPrefix = context
+# number_of_shards = 3
+# number_of_replicas = 0
+# mapping.total_fields.limit = 1000
+# max_docvalue_fields_search = 1000