You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2023/03/07 14:40:46 UTC
[unomi] branch UNOMI-737-indices-reduction-migration updated: UNOMI-737: add integration tests
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch UNOMI-737-indices-reduction-migration
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/UNOMI-737-indices-reduction-migration by this push:
new 8be288576 UNOMI-737: add integration tests
8be288576 is described below
commit 8be288576a5cc1acd8d66ff32309267800f52f2f
Author: Kevan <ke...@jahia.com>
AuthorDate: Tue Mar 7 15:40:37 2023 +0100
UNOMI-737: add integration tests
---
.../unomi/itests/migration/Migrate16xTo220IT.java | 54 ++++++++++++++++------
.../ElasticSearchPersistenceServiceImpl.java | 3 --
.../META-INF/cxs/mappings/personaSession.json | 41 ++++++++++++++++
.../migrate-2.2.0-05-indicesReduction.groovy | 3 +-
4 files changed, 82 insertions(+), 19 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
index 8ae93daf4..9ddbd941a 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
@@ -38,9 +38,13 @@ public class Migrate16xTo220IT extends BaseIT {
private int eventCount = 0;
private int sessionCount = 0;
+ private int systemItemsCount = 0;
private static final int NUMBER_DUPLICATE_SESSIONS = 3;
- private static final int NUMBER_PERSONA_SESSIONS = 2;
+ private static final List<String> oldSystemItemsIndices = Arrays.asList("actionType", "campaign", "campaignevent", "goal",
+ "userList", "propertyType", "scope", "conditionType", "rule", "scoring", "segment", "groovyAction", "topic",
+ "patch", "jsonSchema", "importConfig", "exportConfig", "rulestats");
+
@Override
@Before
public void waitForStartup() throws InterruptedException {
@@ -56,8 +60,9 @@ public class Migrate16xTo220IT extends BaseIT {
}
// Restore the snapshot
HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true", "{}", null);
- fillNumberEventAndSessionBeforeMigration(httpClient);
+ // Get initial counts of items to compare after migration
+ initCounts(httpClient);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -93,6 +98,7 @@ public class Migrate16xTo220IT extends BaseIT {
checkEventTypesNotPersistedAnymore();
checkForMappingUpdates();
checkEventSessionRollover2_2_0();
+ checkIndexReductions2_2_0();
}
/**
@@ -107,17 +113,29 @@ public class Migrate16xTo220IT extends BaseIT {
int newEventcount = 0;
for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-event-0")) {
- JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null));
- newEventcount += jsonNode.get("count").asInt();
+ newEventcount += countItems(eventIndex, null);
}
int newSessioncount = 0;
for (String sessionIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-session-0")) {
- JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + sessionIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null));
- newSessioncount += jsonNode.get("count").asInt();
+ newSessioncount += countItems(sessionIndex, null);
}
Assert.assertEquals(eventCount, newEventcount);
- Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS + NUMBER_PERSONA_SESSIONS, newSessioncount);
+ Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS, newSessioncount);
+ }
+
+ private void checkIndexReductions2_2_0() throws IOException {
+ // new index for system items:
+ Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400", "context-systemitems"));
+
+ // old indices should be removed:
+ for (String oldSystemItemsIndex : oldSystemItemsIndices) {
+ Assert.assertFalse(MigrationUtils.indexExists(httpClient, "http://localhost:9400", oldSystemItemsIndex));
+ }
+
+ // check counts
+ Assert.assertEquals("Expect same number of system items (rules, segments, etc ...) after 2_2_0 migration",
+ systemItemsCount, countItems("context-systemitems", null));
}
/**
@@ -298,21 +316,29 @@ public class Migrate16xTo220IT extends BaseIT {
Assert.assertNull(persistenceService.load(masterProfile, ProfileAlias.class));
}
- private void fillNumberEventAndSessionBeforeMigration(CloseableHttpClient httpClient) {
+ private void initCounts(CloseableHttpClient httpClient) {
try {
for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-event-date")) {
- JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_count", resourceAsString("migration/must_not_match_some_eventype_body.json"), null));
- eventCount += jsonNode.get("count").asInt();
+ eventCount += countItems(eventIndex, resourceAsString("migration/must_not_match_some_eventype_body.json"));
}
- for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-session-date")) {
- JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null));
- sessionCount += jsonNode.get("count").asInt();
+ for (String sessionIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-session-date")) {
+ sessionCount += countItems(sessionIndex, null);
+ }
+
+ for (String oldSystemItemsIndex : oldSystemItemsIndices) {
+ systemItemsCount += countItems(oldSystemItemsIndex, null);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
+ }
-
+ private int countItems(String index, String requestBody) throws IOException {
+ if (requestBody == null) {
+ requestBody = resourceAsString("migration/must_not_match_some_eventype_body.json");
+ }
+ JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400" + "/" + index + "/_count", requestBody, null));
+ return jsonNode.get("count").asInt();
}
}
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 33fa1902c..381f1a139 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -248,9 +248,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
itemTypeIndexNameMap.put("profile", "profile");
itemTypeIndexNameMap.put("persona", "profile");
-
- itemTypeIndexNameMap.put("session", "session");
- itemTypeIndexNameMap.put("personaSession", "session");
}
public void setBundleContext(BundleContext bundleContext) {
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
new file mode 100644
index 000000000..c635e0285
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
@@ -0,0 +1,41 @@
+{
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "duration": {
+ "type": "long"
+ },
+ "timeStamp": {
+ "type": "date"
+ },
+ "lastEventDate": {
+ "type": "date"
+ },
+ "properties": {
+ "properties": {
+ "location": {
+ "type": "geo_point"
+ }
+ }
+ },
+ "size": {
+ "type": "long"
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
index 905ff9339..631f0bced 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
@@ -43,8 +43,7 @@ def indicesToReduce = [
rulestats: [reduceTo: "systemitems", renameId: true],
groovyaction: [reduceTo: "systemitems", renameId: true],
- persona: [reduceTo: "profile", renameId: false],
- personasession: [reduceTo: "session", renameId: false]
+ persona: [reduceTo: "profile", renameId: false]
]
context.performMigrationStep("2.2.0-create-systemItems-index", () -> {