You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/07/22 07:14:06 UTC
[unomi] branch master updated: UNOMI-605: migrate events to 2.0.0 (#462)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new d8d18da0e UNOMI-605: migrate events to 2.0.0 (#462)
d8d18da0e is described below
commit d8d18da0e93ca0880e1520a02ccd4b8d62cad36e
Author: kevan Jahanshahi <ke...@jahia.com>
AuthorDate: Fri Jul 22 09:14:01 2022 +0200
UNOMI-605: migrate events to 2.0.0 (#462)
* UNOMI-605: migrate events to 2.0.0
* UNOMI-605: migrate events to 2.0.0
---
.../unomi/shell/migration/impl/MigrationTo200.java | 64 +---------------
.../shell/migration/utils/MigrationUtils.java | 19 +++++
.../migrate-2.0.0-03-profileReindex.groovy | 2 +-
.../migrate-2.0.0-04-eventsReindex.groovy | 34 +++++++++
.../requestBody/2.0.0/event_delete_by_query.json | 15 ++++
.../resources/requestBody/2.0.0/event_index.json | 87 ++++++++++++++++++++++
.../requestBody/2.0.0/event_migrate.painless | 51 +++++++++++++
...interests.painless => profile_migrate.painless} | 0
8 files changed, 208 insertions(+), 64 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index edd92562e..3e8967ed0 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -61,74 +61,12 @@ public class MigrationTo200 implements Migration {
}
private void doExecute(String indexPrefix) throws IOException {
- Set<String> indexes = getEventIndexes(indexPrefix);
- for (String index : indexes) {
- updateMapping(index);
- }
+ Set<String> indexes = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, indexPrefix + "-event-");
createScopeMapping(indexPrefix);
createScopes(getSetOfScopes(indexes), indexPrefix);
createProfileAliasDocumentsFromProfile();
}
- private void updateMapping(final String indexName) throws IOException {
- HttpPut httpPut = new HttpPut(esAddress + "/" + indexName + "/_mapping");
-
- httpPut.addHeader("Accept", "application/json");
- httpPut.addHeader("Content-Type", "application/json");
-
- String requestBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/updateMapping.json");
-
- httpPut.setEntity(new StringEntity(requestBody));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
- JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
-
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && responseAsJson.has("acknowledged") && responseAsJson
- .getBoolean("acknowledged")) {
- System.out.println("Mapping for index = \"" + indexName + "\" successfully updated.");
-
- copyValueScopeToSourceId(indexName, httpClient);
- } else {
- System.out.println("Update the mapping for index = \"" + indexName + "\" failed.");
- }
- }
- }
-
- private void copyValueScopeToSourceId(final String indexName, final CloseableHttpClient httpClient) throws IOException {
- final HttpPost httpPost = new HttpPost(esAddress + "/" + indexName + "/_update_by_query");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/json");
-
- String requestBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/copyValueScopeToSourceId.json");
-
- httpPost.setEntity(new StringEntity(requestBody));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
-
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName
- + "\" successfully completed. Total: " + responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
- + ".");
- } else {
- System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName + "\" failed.");
- }
- }
- }
-
- private Set<String> getEventIndexes(String indexPrefix) throws IOException {
- try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
- return indexesAsJson.keySet().stream().
- filter(alias -> alias.startsWith(indexPrefix + "-event")).
- collect(Collectors.toSet());
- }
- }
- return Collections.emptySet();
- }
-
private boolean scopeIndexNotExists(String indexPrefix) throws IOException {
final HttpGet httpGet = new HttpGet(esAddress + "/" + indexPrefix + "-scope");
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index f055be2d3..33b0278ed 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -18,7 +18,11 @@ package org.apache.unomi.shell.migration.utils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
import org.json.JSONObject;
import org.osgi.framework.BundleContext;
@@ -29,6 +33,9 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* @author dgaillard
@@ -78,6 +85,18 @@ public class MigrationUtils {
}
}
+ public static Set<String> getIndexesPrefixedBy(CloseableHttpClient httpClient, String esAddress, String prefix) throws IOException {
+ try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(prefix)).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
String newIndexSettings, String painlessScript) throws IOException {
String indexNameCloned = indexName + "-cloned";
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
index 5eaa63e0e..83ab51233 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
@@ -19,4 +19,4 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/profile_index.json");
MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/migrate_existing_interests.painless"))
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"))
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
new file mode 100644
index 000000000..10c64646f
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
@@ -0,0 +1,34 @@
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+String esAddress = migrationConfig.get("esAddress")
+String indexPrefix = migrationConfig.get("indexPrefix")
+
+// Remove all internal events that are no more persisted
+String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
+HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+
+// Reindex the rest of the events
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_index.json");
+String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
+Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, "${indexPrefix}-event-")
+eventIndices.each { eventIndex ->
+ MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript)
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_delete_by_query.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_delete_by_query.json
new file mode 100644
index 000000000..7830da7f5
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_delete_by_query.json
@@ -0,0 +1,15 @@
+{
+ "query": {
+ "bool": {
+ "should": [
+ {"term": {"eventType.keyword": "profileUpdated"}},
+ {"term": {"eventType.keyword": "profileDeleted"}},
+ {"term": {"eventType.keyword": "anonymizeProfile"}},
+ {"term": {"eventType.keyword": "sessionCreated"}},
+ {"term": {"eventType.keyword": "sessionReassigned"}},
+ {"term": {"eventType.keyword": "updateProperties"}},
+ {"term": {"eventType.keyword": "goal"}}
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_index.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_index.json
new file mode 100644
index 000000000..a0e25f945
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_index.json
@@ -0,0 +1,87 @@
+{
+ "settings": {
+ "index": {
+ "number_of_shards": #numberOfShards,
+ "number_of_replicas": #numberOfReplicas,
+ "mapping.total_fields.limit": #mappingTotalFieldsLimit,
+ "max_docvalue_fields_search": #maxDocValueFieldsSearch
+ },
+ "analysis": {
+ "analyzer": {
+ "folding": {
+ "type": "custom",
+ "tokenizer": "keyword",
+ "filter": [
+ "lowercase",
+ "asciifolding"
+ ]
+ }
+ }
+ }
+ },
+ "mappings": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "flattenedProperties": {
+ "type": "flattened"
+ },
+ "timeStamp": {
+ "type": "date"
+ },
+ "target" : {
+ "properties" : {
+ "lastEventDate" : {
+ "type" : "date"
+ },
+ "profile" : {
+ "properties" : {
+ "properties" : {
+ "properties" : {
+ "birthDate" : {
+ "type" : "date"
+ },
+ "firstVisit" : {
+ "type" : "date"
+ },
+ "lastVisit" : {
+ "type" : "date"
+ },
+ "notificationRefreshDate" : {
+ "type" : "date"
+ },
+ "previousVisit" : {
+ "type" : "date"
+ }
+ }
+ },
+ "systemProperties" : {
+ "properties" : {
+
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
new file mode 100644
index 000000000..5c8703dc8
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Handle view events */
+if ('view' == ctx._source.eventType){
+
+ /* Check for interests */
+ if (ctx._source.target != null && ctx._source.target.properties != null && ctx._source.target.properties.interests != null){
+
+ if (ctx._source.flattenedProperties == null) {
+ ctx._source.put('flattenedProperties', new HashMap());
+ }
+ ctx._source.flattenedProperties.put('interests', ctx._source.target.properties.interests);
+ ctx._source.target.properties.remove('interests');
+ }
+
+ /* Check for URL parameters */
+ if (ctx._source.target != null && ctx._source.target.properties != null && ctx._source.target.properties.pageInfo != null &&
+ ctx._source.target.properties.pageInfo.parameters != null){
+
+ if (ctx._source.flattenedProperties == null) {
+ ctx._source.put('flattenedProperties', new HashMap());
+ }
+ ctx._source.flattenedProperties.put('URLParameters', ctx._source.target.properties.pageInfo.parameters);
+ ctx._source.target.properties.pageInfo.remove('parameters');
+ }
+}
+
+/* Handle form events */
+if ('form' == ctx._source.eventType && ctx._source.properties != null){
+
+ if (ctx._source.flattenedProperties == null) {
+ ctx._source.put('flattenedProperties', new HashMap());
+ }
+ ctx._source.flattenedProperties.put('fields', ctx._source.properties);
+ ctx._source.put('properties', new HashMap());
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_migrate.painless
similarity index 100%
rename from tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_migrate.painless