You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/07/21 17:02:16 UTC
[unomi] branch migrateEvents created (now bc3abcd32)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a change to branch migrateEvents
in repository https://gitbox.apache.org/repos/asf/unomi.git
at bc3abcd32 UNOMI-605: migrate events to 2.0.0
This branch includes the following new commits:
new bc3abcd32 UNOMI-605: migrate events to 2.0.0
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-605: migrate events to 2.0.0
Posted by jk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch migrateEvents
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit bc3abcd32df1ff6e899895f7604bfa5cb9d0a6db
Author: Kevan <ke...@jahia.com>
AuthorDate: Thu Jul 21 19:01:59 2022 +0200
UNOMI-605: migrate events to 2.0.0
---
.../unomi/shell/migration/impl/MigrationTo200.java | 64 +---------------
.../shell/migration/utils/MigrationUtils.java | 19 +++++
.../migrate-2.0.0-03-profileReindex.groovy | 2 +-
.../migrate-2.0.0-04-eventsReindex.groovy | 34 +++++++++
.../requestBody/2.0.0/event_delete_by_query.json | 15 ++++
.../resources/requestBody/2.0.0/event_index.json | 87 ++++++++++++++++++++++
.../requestBody/2.0.0/event_migrate.painless | 55 ++++++++++++++
...interests.painless => profile_migrate.painless} | 0
8 files changed, 212 insertions(+), 64 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
index edd92562e..3e8967ed0 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
@@ -61,74 +61,12 @@ public class MigrationTo200 implements Migration {
}
private void doExecute(String indexPrefix) throws IOException {
- Set<String> indexes = getEventIndexes(indexPrefix);
- for (String index : indexes) {
- updateMapping(index);
- }
+ Set<String> indexes = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, indexPrefix + "-event-");
createScopeMapping(indexPrefix);
createScopes(getSetOfScopes(indexes), indexPrefix);
createProfileAliasDocumentsFromProfile();
}
- private void updateMapping(final String indexName) throws IOException {
- HttpPut httpPut = new HttpPut(esAddress + "/" + indexName + "/_mapping");
-
- httpPut.addHeader("Accept", "application/json");
- httpPut.addHeader("Content-Type", "application/json");
-
- String requestBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/updateMapping.json");
-
- httpPut.setEntity(new StringEntity(requestBody));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
- JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
-
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && responseAsJson.has("acknowledged") && responseAsJson
- .getBoolean("acknowledged")) {
- System.out.println("Mapping for index = \"" + indexName + "\" successfully updated.");
-
- copyValueScopeToSourceId(indexName, httpClient);
- } else {
- System.out.println("Update the mapping for index = \"" + indexName + "\" failed.");
- }
- }
- }
-
- private void copyValueScopeToSourceId(final String indexName, final CloseableHttpClient httpClient) throws IOException {
- final HttpPost httpPost = new HttpPost(esAddress + "/" + indexName + "/_update_by_query");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/json");
-
- String requestBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/copyValueScopeToSourceId.json");
-
- httpPost.setEntity(new StringEntity(requestBody));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
-
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName
- + "\" successfully completed. Total: " + responseAsJson.get("total") + ", updated: " + responseAsJson.get("updated")
- + ".");
- } else {
- System.out.println("Copying the \"scope\" field to the \"sourceId\" field for index = \"" + indexName + "\" failed.");
- }
- }
- }
-
- private Set<String> getEventIndexes(String indexPrefix) throws IOException {
- try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
- return indexesAsJson.keySet().stream().
- filter(alias -> alias.startsWith(indexPrefix + "-event")).
- collect(Collectors.toSet());
- }
- }
- return Collections.emptySet();
- }
-
private boolean scopeIndexNotExists(String indexPrefix) throws IOException {
final HttpGet httpGet = new HttpGet(esAddress + "/" + indexPrefix + "-scope");
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index f055be2d3..33b0278ed 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -18,7 +18,11 @@ package org.apache.unomi.shell.migration.utils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
import org.json.JSONObject;
import org.osgi.framework.BundleContext;
@@ -29,6 +33,9 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
/**
* @author dgaillard
@@ -78,6 +85,18 @@ public class MigrationUtils {
}
}
+ public static Set<String> getIndexesPrefixedBy(CloseableHttpClient httpClient, String esAddress, String prefix) throws IOException {
+ try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
+ if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
+ JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
+ return indexesAsJson.keySet().stream().
+ filter(alias -> alias.startsWith(prefix)).
+ collect(Collectors.toSet());
+ }
+ }
+ return Collections.emptySet();
+ }
+
public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName,
String newIndexSettings, String painlessScript) throws IOException {
String indexNameCloned = indexName + "-cloned";
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
index 5eaa63e0e..83ab51233 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
@@ -19,4 +19,4 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils
String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/profile_index.json");
MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-profile",
- newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/migrate_existing_interests.painless"))
+ newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"))
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
new file mode 100644
index 000000000..10c64646f
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
@@ -0,0 +1,34 @@
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+String esAddress = migrationConfig.get("esAddress")
+String indexPrefix = migrationConfig.get("indexPrefix")
+
+// Remove all internal events that are no more persisted
+String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
+HttpUtils.executePostRequest(httpClient, "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+
+// Reindex the rest of the events
+String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_index.json");
+String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
+Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, "${indexPrefix}-event-")
+eventIndices.each { eventIndex ->
+ MigrationUtils.reIndex(httpClient, bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript)
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_delete_by_query.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_delete_by_query.json
new file mode 100644
index 000000000..7830da7f5
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_delete_by_query.json
@@ -0,0 +1,15 @@
+{
+ "query": {
+ "bool": {
+ "should": [
+ {"term": {"eventType.keyword": "profileUpdated"}},
+ {"term": {"eventType.keyword": "profileDeleted"}},
+ {"term": {"eventType.keyword": "anonymizeProfile"}},
+ {"term": {"eventType.keyword": "sessionCreated"}},
+ {"term": {"eventType.keyword": "sessionReassigned"}},
+ {"term": {"eventType.keyword": "updateProperties"}},
+ {"term": {"eventType.keyword": "goal"}}
+ ]
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_index.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_index.json
new file mode 100644
index 000000000..a0e25f945
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_index.json
@@ -0,0 +1,87 @@
+{
+ "settings": {
+ "index": {
+ "number_of_shards": #numberOfShards,
+ "number_of_replicas": #numberOfReplicas,
+ "mapping.total_fields.limit": #mappingTotalFieldsLimit,
+ "max_docvalue_fields_search": #maxDocValueFieldsSearch
+ },
+ "analysis": {
+ "analyzer": {
+ "folding": {
+ "type": "custom",
+ "tokenizer": "keyword",
+ "filter": [
+ "lowercase",
+ "asciifolding"
+ ]
+ }
+ }
+ }
+ },
+ "mappings": {
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "flattenedProperties": {
+ "type": "flattened"
+ },
+ "timeStamp": {
+ "type": "date"
+ },
+ "target" : {
+ "properties" : {
+ "lastEventDate" : {
+ "type" : "date"
+ },
+ "profile" : {
+ "properties" : {
+ "properties" : {
+ "properties" : {
+ "birthDate" : {
+ "type" : "date"
+ },
+ "firstVisit" : {
+ "type" : "date"
+ },
+ "lastVisit" : {
+ "type" : "date"
+ },
+ "notificationRefreshDate" : {
+ "type" : "date"
+ },
+ "previousVisit" : {
+ "type" : "date"
+ }
+ }
+ },
+ "systemProperties" : {
+ "properties" : {
+
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
new file mode 100644
index 000000000..4fd878064
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Handle view events */
+if ('view' == ctx._source.eventType){
+
+ /* Check for interests */
+ if (ctx._source.target != null && ctx._source.target.properties != null && ctx._source.target.properties.interests != null){
+
+ if (ctx._source.flattenedProperties == null) {
+ ctx._source.put('flattenedProperties', new HashMap());
+ }
+ ctx._source.flattenedProperties.put('interests', ctx._source.target.properties.interests);
+ ctx._source.target.properties.remove('interests');
+ }
+
+ /* Check for URL parameters */
+ if (ctx._source.target != null && ctx._source.target.properties != null && ctx._source.target.properties.pageInfo != null &&
+ ctx._source.target.properties.pageInfo.parameters != null){
+
+ if (ctx._source.flattenedProperties == null) {
+ ctx._source.put('flattenedProperties', new HashMap());
+ }
+ ctx._source.flattenedProperties.put('URLParameters', ctx._source.target.properties.pageInfo.parameters);
+ ctx._source.target.properties.pageInfo.remove('parameters');
+ }
+}
+
+/* Handle form events */
+if ('form' == ctx._source.eventType){
+
+ /* Check for form fields */
+ if (ctx._source.properties != null){
+
+ if (ctx._source.flattenedProperties == null) {
+ ctx._source.put('flattenedProperties', new HashMap());
+ }
+ ctx._source.flattenedProperties.put('fields', ctx._source.properties);
+ ctx._source.put('properties', new HashMap());
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_migrate.painless
similarity index 100%
rename from tools/shell-commands/src/main/resources/requestBody/2.0.0/migrate_existing_interests.painless
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_migrate.painless