You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by dg...@apache.org on 2022/08/25 11:54:17 UTC
[unomi] branch master updated: UNOMI-644 : update migration script to add scope to login event (#479)
This is an automated email from the ASF dual-hosted git repository.
dgriffon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new fbc0e0d66 UNOMI-644 : update migration script to add scope to login event (#479)
fbc0e0d66 is described below
commit fbc0e0d660550551d07f68bdc35fba132eafaa79
Author: David Griffon <dg...@jahia.com>
AuthorDate: Thu Aug 25 13:54:13 2022 +0200
UNOMI-644 : update migration script to add scope to login event (#479)
* UNOMI-644 : update migration script to add scope to login event
* UNOMI-644 : add info to test
* UNOMI-644 : perform a refresh on index after migration, wait for Yellow status of the index before finishing reindex step.
---
.../unomi/itests/migration/Migrate16xTo200IT.java | 20 +++++++++++++-
.../resources/migration/snapshots_repository.zip | Bin 880112 -> 868699 bytes
.../shell/migration/service/MigrationContext.java | 2 +-
.../shell/migration/utils/MigrationUtils.java | 29 ++++++++++++++++-----
...copes.groovy => migrate-2.0.0-20-scopes.groovy} | 0
.../requestBody/2.0.0/event_migrate.painless | 11 +++++++-
6 files changed, 52 insertions(+), 10 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
index 115182ed8..6666f43f0 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
@@ -79,6 +79,7 @@ public class Migrate16xTo200IT extends BaseIT {
checkMergedProfilesAliases();
checkProfileInterests();
checkScopeHaveBeenCreated();
+ checkLoginEventWithScope();
checkFormEventRestructured();
checkViewEventRestructured();
checkEventTypesNotPersistedAnymore();
@@ -145,6 +146,23 @@ public class Migrate16xTo200IT extends BaseIT {
}
}
+ private void checkLoginEventWithScope() {
+ List<Event> events = persistenceService.query("eventType", "view", null, Event.class);
+ List<String> digitallLoginEvent = Arrays.asList("4054a3e0-35ef-4256-999b-b9c05c1209f1", "f3f71ff8-2d6d-4b6c-8bdc-cb39905cddfe", "ff24ae6f-5a98-421e-aeb0-e86855b462ff");
+ for (Event loginEvent : events) {
+ if (loginEvent.getItemId().equals("5c4ac1df-f42b-4117-9432-12fdf9ecdf98")) {
+ Assert.assertEquals(loginEvent.getScope(), "systemsite");
+ Assert.assertEquals(loginEvent.getTarget().getScope(), "systemsite");
+ Assert.assertEquals(loginEvent.getSource().getScope(), "systemsite");
+ }
+ if (digitallLoginEvent.contains(loginEvent.getItemId())) {
+ Assert.assertEquals(loginEvent.getScope(), "digitall");
+ Assert.assertEquals(loginEvent.getTarget().getScope(), "digitall");
+ Assert.assertEquals(loginEvent.getSource().getScope(), "digitall");
+ }
+ }
+ }
+
/**
* Data set contains a view event (id: a4aa836b-c437-48ef-be02-6fbbcba3a1de) with two interests: football:50 and basketball:30
* Data set contains a view event (id: 34d53399-f173-451f-8d48-f34f5d9618a9) with two URL Parameters: paramerter_test:value, multiple_paramerter_test:[value1, value2]
@@ -199,7 +217,7 @@ public class Migrate16xTo200IT extends BaseIT {
for (String scopeFromEvents : existingScopesFromEvents.keySet()) {
if (!Objects.equals(scopeFromEvents, "_filtered")) {
Scope scope = scopeService.getScope(scopeFromEvents);
- Assert.assertNotNull(scope);
+ Assert.assertNotNull(String.format("Unable to find registered scope %s", scopeFromEvents), scope);
}
}
}
diff --git a/itests/src/test/resources/migration/snapshots_repository.zip b/itests/src/test/resources/migration/snapshots_repository.zip
index 58eb87e72..1252d50a1 100644
Binary files a/itests/src/test/resources/migration/snapshots_repository.zip and b/itests/src/test/resources/migration/snapshots_repository.zip differ
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
index 2f20353c2..21835c31e 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationContext.java
@@ -179,7 +179,7 @@ public class MigrationContext {
logger.info(msg);
} else {
PrintStream writer = session.getConsole();
- writer.println(msg);
+ writer.printf("%s: %s%n",new Date(), msg);
}
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index a57d79dec..40fd9f549 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -29,11 +29,7 @@ import org.json.JSONObject;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.*;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
@@ -125,7 +121,7 @@ public class MigrationUtils {
throw new RuntimeException("no mapping found in bundles for: " + fileName);
}
- public static String buildIndexCreationRequest(String baseIndexSettings, String mapping, MigrationContext context, boolean isMonthlyIndex) throws IOException {
+ public static String buildIndexCreationRequest(String baseIndexSettings, String mapping, MigrationContext context, boolean isMonthlyIndex) throws IOException {
String settings = baseIndexSettings
.replace("#numberOfShards", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_SHARDS : NUMBER_OF_SHARDS))
.replace("#numberOfReplicas", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_REPLICAS : NUMBER_OF_REPLICAS))
@@ -178,6 +174,10 @@ public class MigrationUtils {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
}
});
+ // Do a refresh
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null);
+
+ waitForYellowStatus(httpClient, esAddress, migrationContext);
}
public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
@@ -185,7 +185,7 @@ public class MigrationUtils {
while (true) {
JSONObject responseAsJson = new JSONObject(response);
- String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id"): null;
+ String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id") : null;
JSONArray hits = new JSONArray();
if (responseAsJson.has("hits")) {
JSONObject hitsObject = responseAsJson.getJSONObject("hits");
@@ -215,6 +215,21 @@ public class MigrationUtils {
}
}
+ /**
+ * Utility method that waits for the ES cluster to be in yellow status
+ */
+ public static void waitForYellowStatus(CloseableHttpClient httpClient, String esAddress, MigrationContext migrationContext) throws Exception {
+ while (true) {
+ final JSONObject status = new JSONObject(HttpUtils.executeGetRequest(httpClient, esAddress + "/_cluster/health?wait_for_status=yellow&timeout=60s", null));
+ if (!status.get("timed_out").equals("true")) {
+ migrationContext.printMessage("ES Cluster status is " + status.get("status"));
+ break;
+ }
+ migrationContext.printMessage("Waiting for ES Cluster status to be Yellow, current status is " + status.get("status"));
+ }
+
+ }
+
public interface ScrollCallback {
void execute(String hits);
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
index 5c8703dc8..800fa97b7 100644
--- a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless
@@ -15,8 +15,17 @@
* limitations under the License.
*/
+/* Handle login events */
+if ('login' == ctx._source.eventType) {
+ /* Look for empty scope */
+ if (ctx._source.scope == '') {
+ ctx._source.put('scope', 'systemsite');
+ ctx._source.source.put('scope', 'systemsite');
+ ctx._source.target.put('scope', 'systemsite');
+ }
+}
/* Handle view events */
-if ('view' == ctx._source.eventType){
+if ('view' == ctx._source.eventType) {
/* Check for interests */
if (ctx._source.target != null && ctx._source.target.properties != null && ctx._source.target.properties.interests != null){