You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2022/07/11 14:36:07 UTC
[unomi] branch master updated: UNOMI-606 : create migration for scoring and fix issues (#460)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new ae7afc419 UNOMI-606 : create migration for scoring and fix issues (#460)
ae7afc419 is described below
commit ae7afc419c2cfb9e801b79dd276c275490aeee9e
Author: jsinovassin <58...@users.noreply.github.com>
AuthorDate: Mon Jul 11 16:36:03 2022 +0200
UNOMI-606 : create migration for scoring and fix issues (#460)
* UNOMI-606 : create migration for scoring and fix issues
---
.../unomi/shell/migration/MigrationScript.java | 2 +-
.../unomi/shell/migration/actions/Migrate.java | 6 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 2 +-
.../unomi/shell/migration/impl/MigrationTo200.java | 5 +-
.../unomi/shell/migration/utils/ConsoleUtils.java | 6 +-
.../shell/migration/utils/MigrationUtils.java | 1 -
.../migrate-2.0.0-01-segmentReindex.groovy | 3 +-
... => migrate-2.0.0-02-scoringPlanReindex.groovy} | 5 +-
.../resources/requestBody/2.0.0/scoring_index.json | 68 ++++++++++++++++++++++
9 files changed, 82 insertions(+), 16 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
index d2dc304ed..0355781bb 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
@@ -39,7 +39,7 @@ import java.util.regex.Pattern;
*/
public class MigrationScript implements Comparable<MigrationScript> {
- private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-(\\w+).groovy$");
+ private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-([\\w|.]+).groovy$");
private final String script;
private Script compiledScript;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index d325cee56..80778ec2b 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -46,6 +46,7 @@ import java.util.stream.Stream;
public class Migrate implements Action {
public static final String CONFIG_ES_ADDRESS = "esAddress";
public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
+ public static final String INDEX_PREFIX = "indexPrefix";
@Reference
Session session;
@@ -53,7 +54,7 @@ public class Migrate implements Action {
@Reference
BundleContext bundleContext;
- @Argument(index = 0, name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
+ @Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
private String originVersion;
public Object execute() throws Exception {
@@ -90,6 +91,7 @@ public class Migrate implements Action {
Map<String, Object> migrationConfig = new HashMap<>();
migrationConfig.put(CONFIG_ES_ADDRESS, ConsoleUtils.askUserWithDefaultAnswer(session, "Enter ElasticSearch 7 TARGET address (default = http://localhost:9200): ", "http://localhost:9200"));
migrationConfig.put(CONFIG_TRUST_ALL_CERTIFICATES, ConsoleUtils.askUserWithAuthorizedAnswer(session,"We need to initialize a HttpClient, do we need to trust all certificates? (yes/no): ", Arrays.asList("yes", "no")).equalsIgnoreCase("yes"));
+ migrationConfig.put(INDEX_PREFIX, ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context"));
try (CloseableHttpClient httpClient = HttpUtils.initHttpClient((Boolean) migrationConfig.get(CONFIG_TRUST_ALL_CERTIFICATES))) {
@@ -107,7 +109,7 @@ public class Migrate implements Action {
return null;
}
- ConsoleUtils.printMessage(session, "Finnish execution of: " + migrateScript);
+ ConsoleUtils.printMessage(session, "Finish execution of: " + migrateScript);
}
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index f0aee214e..4eb84c821 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -40,7 +40,7 @@ public class MigrationTo150 implements Migration {
public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException {
String esAddress = (String) migrationConfig.get("esAddress");
String es5Address = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
- String sourceIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context");
+ String sourceIndexPrefix = (String) migrationConfig.get("indexPrefix");
String destIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "TARGET index prefix (default: context) : ", "context");
int numberOfShards = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of shards for TARGET (default: 5) : ", "5"));
int numberOfReplicas = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of replicas for TARGET (default: 1) : ", "1"));
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index a859bfb83..6b94e16b7 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -57,11 +57,10 @@ public class MigrationTo200 implements Migration {
this.esAddress = (String) migrationConfig.get("esAddress");
this.bundleContext = bundleContext;
- doExecute();
+ doExecute((String) migrationConfig.get("indexPrefix"));
}
- private void doExecute() throws IOException {
- String indexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context");
+ private void doExecute(String indexPrefix) throws IOException {
Set<String> indexes = getEventIndexes(indexPrefix);
for (String index : indexes) {
updateMapping(index);
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
index 145e24921..b56a7568c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
@@ -19,8 +19,6 @@ package org.apache.unomi.shell.migration.utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.karaf.shell.api.console.Session;
import org.jline.reader.LineReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintStream;
@@ -30,7 +28,6 @@ import java.util.List;
* @author dgaillard
*/
public class ConsoleUtils {
- private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
/**
* This will ask a question to the user and return the default answer if the user does not answer.
@@ -74,9 +71,8 @@ public class ConsoleUtils {
* @param session the shell's session
* @param msg String message to prompt
* @return the user answer
- * @throws IOException if there was an error retrieving an answer from the user on the console
*/
- public static String promptMessageToUser(Session session, String msg) throws IOException {
+ public static String promptMessageToUser(Session session, String msg) {
LineReader reader = (LineReader) session.get(".jline.reader");
return reader.readLine(msg, null);
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 22e1d3782..6b93dd0a0 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -64,7 +64,6 @@ public class MigrationUtils {
// Init requests
JSONObject originalIndexSettings = new JSONObject(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + indexName + "/_settings", null));
- // TODO UNOMI-606 validate following lines: (normally those properties are automatically added to unomi indices so they should always be present on the existing indices)
String newIndexRequest = newIndexSettings
.replace("#numberOfShards", originalIndexSettings.getJSONObject(indexName).getJSONObject("settings").getJSONObject("index").getString("number_of_shards"))
.replace("#numberOfReplicas", originalIndexSettings.getJSONObject(indexName).getJSONObject("settings").getJSONObject("index").getString("number_of_replicas"))
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
index 1ffe88aaa..01a2be0c6 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
@@ -18,4 +18,5 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
*/
String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), "context-segment", newIndexSettings)
\ No newline at end of file
+MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment",
+ newIndexSettings)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
similarity index 87%
copy from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
copy to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
index 1ffe88aaa..f30f2ae6f 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
@@ -17,5 +17,6 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), "context-segment", newIndexSettings)
\ No newline at end of file
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scoring_index.json");
+MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-scoring",
+ newIndexSettings)
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json
new file mode 100644
index 000000000..d2541861d
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json
@@ -0,0 +1,68 @@
+{
+ "settings": {
+ "index": {
+ "number_of_shards": #numberOfShards,
+ "number_of_replicas": #numberOfReplicas,
+ "mapping.total_fields.limit": #mappingTotalFieldsLimit,
+ "max_docvalue_fields_search": #maxDocValueFieldsSearch
+ },
+ "analysis": {
+ "analyzer": {
+ "folding": {
+ "type": "custom",
+ "tokenizer": "keyword",
+ "filter": [
+ "lowercase",
+ "asciifolding"
+ ]
+ }
+ }
+ }
+ },
+ "mappings": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "metadata": {
+ "properties": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "hidden": {
+ "type": "boolean"
+ },
+ "missingPlugins": {
+ "type": "boolean"
+ },
+ "readOnly": {
+ "type": "boolean"
+ }
+ }
+ },
+ "elements": {
+ "properties": {
+ "condition": {
+ "type": "object",
+ "enabled": false
+ }
+ }
+ }
+ }
+ }
+}