You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2022/07/11 11:49:54 UTC
[unomi] branch UNOMI-606-migration-segments-scoring created (now 5279bd3c0)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a change to branch UNOMI-606-migration-segments-scoring
in repository https://gitbox.apache.org/repos/asf/unomi.git
at 5279bd3c0 UNOMI-606 : create migration for scoring and fix issues
This branch includes the following new commits:
new 5279bd3c0 UNOMI-606 : create migration for scoring and fix issues
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-606 : create migration for scoring and fix issues
Posted by js...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch UNOMI-606-migration-segments-scoring
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 5279bd3c0d86b8f75e8b72198fb05c5ea3feba47
Author: jsinovassin <js...@jahia.com>
AuthorDate: Mon Jul 11 13:49:41 2022 +0200
UNOMI-606 : create migration for scoring and fix issues
---
.../unomi/shell/migration/MigrationScript.java | 2 +-
.../unomi/shell/migration/actions/Migrate.java | 4 +-
.../unomi/shell/migration/impl/MigrationTo150.java | 2 +-
.../unomi/shell/migration/impl/MigrationTo200.java | 5 +-
.../unomi/shell/migration/utils/ConsoleUtils.java | 6 +-
.../migrate-2.0.0-01-segmentReindex.groovy | 3 +-
... => migrate-2.0.0-02-scoringPlanReindex.groovy} | 5 +-
.../resources/requestBody/2.0.0/scoring_index.json | 68 ++++++++++++++++++++++
8 files changed, 81 insertions(+), 14 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
index d2dc304ed..0355781bb 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java
@@ -39,7 +39,7 @@ import java.util.regex.Pattern;
*/
public class MigrationScript implements Comparable<MigrationScript> {
- private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-(\\w+).groovy$");
+ private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-([\\w|.]+).groovy$");
private final String script;
private Script compiledScript;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index d325cee56..a3bf07837 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -46,6 +46,7 @@ import java.util.stream.Stream;
public class Migrate implements Action {
public static final String CONFIG_ES_ADDRESS = "esAddress";
public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates";
+ public static final String INDEX_PREFIX = "indexPrefix";
@Reference
Session session;
@@ -53,7 +54,7 @@ public class Migrate implements Action {
@Reference
BundleContext bundleContext;
- @Argument(index = 0, name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
+ @Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0")
private String originVersion;
public Object execute() throws Exception {
@@ -90,6 +91,7 @@ public class Migrate implements Action {
Map<String, Object> migrationConfig = new HashMap<>();
migrationConfig.put(CONFIG_ES_ADDRESS, ConsoleUtils.askUserWithDefaultAnswer(session, "Enter ElasticSearch 7 TARGET address (default = http://localhost:9200): ", "http://localhost:9200"));
migrationConfig.put(CONFIG_TRUST_ALL_CERTIFICATES, ConsoleUtils.askUserWithAuthorizedAnswer(session,"We need to initialize a HttpClient, do we need to trust all certificates? (yes/no): ", Arrays.asList("yes", "no")).equalsIgnoreCase("yes"));
+ migrationConfig.put(INDEX_PREFIX, ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context"));
try (CloseableHttpClient httpClient = HttpUtils.initHttpClient((Boolean) migrationConfig.get(CONFIG_TRUST_ALL_CERTIFICATES))) {
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
index f0aee214e..4eb84c821 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java
@@ -40,7 +40,7 @@ public class MigrationTo150 implements Migration {
public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException {
String esAddress = (String) migrationConfig.get("esAddress");
String es5Address = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210");
- String sourceIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context");
+ String sourceIndexPrefix = (String) migrationConfig.get("indexPrefix");
String destIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "TARGET index prefix (default: context) : ", "context");
int numberOfShards = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of shards for TARGET (default: 5) : ", "5"));
int numberOfReplicas = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of replicas for TARGET (default: 1) : ", "1"));
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index a859bfb83..6b94e16b7 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -57,11 +57,10 @@ public class MigrationTo200 implements Migration {
this.esAddress = (String) migrationConfig.get("esAddress");
this.bundleContext = bundleContext;
- doExecute();
+ doExecute((String) migrationConfig.get("indexPrefix"));
}
- private void doExecute() throws IOException {
- String indexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context");
+ private void doExecute(String indexPrefix) throws IOException {
Set<String> indexes = getEventIndexes(indexPrefix);
for (String index : indexes) {
updateMapping(index);
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
index 145e24921..b56a7568c 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java
@@ -19,8 +19,6 @@ package org.apache.unomi.shell.migration.utils;
import org.apache.commons.lang3.StringUtils;
import org.apache.karaf.shell.api.console.Session;
import org.jline.reader.LineReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.PrintStream;
@@ -30,7 +28,6 @@ import java.util.List;
* @author dgaillard
*/
public class ConsoleUtils {
- private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class);
/**
* This will ask a question to the user and return the default answer if the user does not answer.
@@ -74,9 +71,8 @@ public class ConsoleUtils {
* @param session the shell's session
* @param msg String message to prompt
* @return the user answer
- * @throws IOException if there was an error retrieving an answer from the user on the console
*/
- public static String promptMessageToUser(Session session, String msg) throws IOException {
+ public static String promptMessageToUser(Session session, String msg) {
LineReader reader = (LineReader) session.get(".jline.reader");
return reader.readLine(msg, null);
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
index 1ffe88aaa..01a2be0c6 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
@@ -18,4 +18,5 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
*/
String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), "context-segment", newIndexSettings)
\ No newline at end of file
+MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment",
+ newIndexSettings)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
similarity index 87%
copy from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
copy to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
index 1ffe88aaa..f30f2ae6f 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
@@ -17,5 +17,6 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), "context-segment", newIndexSettings)
\ No newline at end of file
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scoring_index.json");
+MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-scoring",
+ newIndexSettings)
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json
new file mode 100644
index 000000000..d2541861d
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json
@@ -0,0 +1,68 @@
+{
+ "settings": {
+ "index": {
+ "number_of_shards": #numberOfShards,
+ "number_of_replicas": #numberOfReplicas,
+ "mapping.total_fields.limit": #mappingTotalFieldsLimit,
+ "max_docvalue_fields_search": #maxDocValueFieldsSearch
+ },
+ "analysis": {
+ "analyzer": {
+ "folding": {
+ "type": "custom",
+ "tokenizer": "keyword",
+ "filter": [
+ "lowercase",
+ "asciifolding"
+ ]
+ }
+ }
+ }
+ },
+ "mappings": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "metadata": {
+ "properties": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "hidden": {
+ "type": "boolean"
+ },
+ "missingPlugins": {
+ "type": "boolean"
+ },
+ "readOnly": {
+ "type": "boolean"
+ }
+ }
+ },
+ "elements": {
+ "properties": {
+ "condition": {
+ "type": "object",
+ "enabled": false
+ }
+ }
+ }
+ }
+ }
+}