You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2022/07/12 16:34:25 UTC
[unomi] branch master updated: Unomi 603 migration profiles (#461)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 57179ecec Unomi 603 migration profiles (#461)
57179ecec is described below
commit 57179ecec97d861a72d08d48983f0b921d11e35e
Author: jsinovassin <58...@users.noreply.github.com>
AuthorDate: Tue Jul 12 18:34:20 2022 +0200
Unomi 603 migration profiles (#461)
* UNOMI-603 : create migration for profile and fix issues
---
.../unomi/shell/migration/impl/MigrationTo200.java | 3 +-
.../shell/migration/utils/MigrationUtils.java | 37 +++++++++--
.../migrate-2.0.0-01-segmentReindex.groovy | 2 +-
.../migrate-2.0.0-02-scoringPlanReindex.groovy | 2 +-
...oovy => migrate-2.0.0-03-profileReindex.groovy} | 6 +-
.../requestBody/2.0.0/base_reindex_request.json | 4 +-
.../2.0.0/migrate_existing_interests.painless} | 17 +++--
.../resources/requestBody/2.0.0/profile_index.json | 76 ++++++++++++++++++++++
8 files changed, 128 insertions(+), 19 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index 6b94e16b7..edd92562e 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -243,7 +243,8 @@ public class MigrationTo200 implements Migration {
while (true) {
JSONObject responseAsJson = getResponseAsJSON(response);
- String scrollId = responseAsJson.getString("_scroll_id");
+
+ String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id"): null;
JSONArray hits = getProfileHits(responseAsJson);
if (hits.length() == 0) {
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 6b93dd0a0..f055be2d3 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -17,14 +17,16 @@
package org.apache.unomi.shell.migration.utils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.json.JSONObject;
import org.osgi.framework.BundleContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@@ -32,7 +34,6 @@ import java.nio.charset.StandardCharsets;
* @author dgaillard
*/
public class MigrationUtils {
- private static final Logger logger = LoggerFactory.getLogger(MigrationUtils.class);
public static JSONObject queryWithScroll(CloseableHttpClient httpClient, String url) throws IOException {
url += "?scroll=1m";
@@ -59,7 +60,26 @@ public class MigrationUtils {
}
}
- public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName, String newIndexSettings) throws IOException {
+ public static String getFileWithoutComments(BundleContext bundleContext, final String resource) {
+ final URL url = bundleContext.getBundle().getResource(resource);
+ try (InputStream stream = url.openStream()) {
+ DataInputStream in = new DataInputStream(stream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String line;
+ StringBuilder value = new StringBuilder();
+ while ((line = br.readLine()) != null) {
+ if (!line.startsWith("/*") && !line.startsWith(" *") && !line.startsWith("*/"))
+ value.append(line);
+ }
+ in.close();
+ return value.toString();
+ } catch (final Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
+ String newIndexSettings, String painlessScript) throws IOException {
String indexNameCloned = indexName + "-cloned";
// Init requests
@@ -70,8 +90,9 @@ public class MigrationUtils {
.replace("#maxDocValueFieldsSearch", originalIndexSettings.getJSONObject(indexName).getJSONObject("settings").getJSONObject("index").getString("max_docvalue_fields_search"))
.replace("#mappingTotalFieldsLimit", originalIndexSettings.getJSONObject(indexName).getJSONObject("settings").getJSONObject("index").getJSONObject("mapping").getJSONObject("total_fields").getString("limit"));
String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json")
- .replace("#source", indexNameCloned)
- .replace("#dest", indexName);
+ .replace("#source", indexNameCloned).replace("#dest", indexName)
+ .replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : "");
+
String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json");
// Set original index as readOnly
@@ -87,4 +108,8 @@ public class MigrationUtils {
// Remove clone
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
}
+
+ private static String getScriptPart(String painlessScript) {
+ return ", \"script\": {\"source\": \"" + painlessScript + "\", \"lang\": \"painless\"}";
+ }
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
index 01a2be0c6..96048e37f 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
@@ -19,4 +19,4 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment",
- newIndexSettings)
+ newIndexSettings, null)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
index f30f2ae6f..46e898117 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy
@@ -19,4 +19,4 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scoring_index.json");
MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-scoring",
- newIndexSettings)
+ newIndexSettings, null)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
similarity index 79%
copy from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
copy to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
index 01a2be0c6..5eaa63e0e 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
@@ -17,6 +17,6 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment",
- newIndexSettings)
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/profile_index.json");
+MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-profile",
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/migrate_existing_interests.painless"))
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json
index bf7406131..589e71084 100644
--- a/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json
@@ -4,5 +4,5 @@
},
"dest": {
"index": "#dest"
- }
-}
\ No newline at end of file
+ }#painless
+}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless
similarity index 63%
copy from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
copy to tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless
index 01a2be0c6..93fd9cec6 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless
@@ -1,5 +1,3 @@
-import org.apache.unomi.shell.migration.utils.MigrationUtils
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,6 +15,15 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
* limitations under the License.
*/
-String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json");
-MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment",
- newIndexSettings)
+List interests = new ArrayList();
+
+if (ctx._source.properties.interests != null && !(ctx._source.properties.interests instanceof List)){
+ for (entry in ctx._source.properties.interests.entrySet()){
+ Map interest = new HashMap();
+ interest.put('key',entry.getKey());
+ interest.put('value', entry.getValue());
+ interests.add(interest);
+ }
+ ctx._source.properties.remove('interests');
+ ctx._source.properties.put('interests',interests);
+}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_index.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_index.json
new file mode 100644
index 000000000..0808aa03e
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_index.json
@@ -0,0 +1,76 @@
+{
+ "settings": {
+ "index": {
+ "number_of_shards": #numberOfShards,
+ "number_of_replicas": #numberOfReplicas,
+ "mapping.total_fields.limit": #mappingTotalFieldsLimit,
+ "max_docvalue_fields_search": #maxDocValueFieldsSearch
+ },
+ "analysis": {
+ "analyzer": {
+ "folding": {
+ "type": "custom",
+ "tokenizer": "keyword",
+ "filter": [
+ "lowercase",
+ "asciifolding"
+ ]
+ }
+ }
+ }
+ },
+ "mappings": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "properties": {
+ "properties": {
+ "age": {
+ "type": "long"
+ },
+ "firstVisit": {
+ "type": "date"
+ },
+ "lastVisit": {
+ "type": "date"
+ },
+ "previousVisit": {
+ "type": "date"
+ },
+ "nbOfVisits": {
+ "type": "long"
+ },
+ "interests": {
+ "type": "nested"
+ }
+ }
+ },
+ "consents": {
+ "properties": {
+ "statusDate": {
+ "type": "date"
+ },
+ "revokeDate": {
+ "type": "date"
+ }
+ }
+ }
+ }
+ }
+}