You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/08 16:19:13 UTC
[unomi] branch eventCollectorSessionProfileHandling updated: UNOMI-628, UNOMI-629: Cleanup profile alias mechanism, provide better migration and itest on the migrated data
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch eventCollectorSessionProfileHandling
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/eventCollectorSessionProfileHandling by this push:
new 1352354c7 UNOMI-628, UNOMI-629: Cleanup profile alias mechanism, provide better migration and itest on the migrated data
1352354c7 is described below
commit 1352354c7e336bc7e08400f8691babedf9918055
Author: Kevan <ke...@jahia.com>
AuthorDate: Mon Aug 8 18:18:56 2022 +0200
UNOMI-628, UNOMI-629: Cleanup profile alias mechanism, provide better migration and itest on the migrated data
---
.../main/java/org/apache/unomi/api/Profile.java | 14 +-
.../karaf-kar/src/main/feature/feature.xml | 5 +-
extensions/groovy-actions/services/pom.xml | 6 +
.../test/java/org/apache/unomi/itests/AllITs.java | 34 +--
.../org/apache/unomi/itests/ProfileServiceIT.java | 2 +
.../unomi/itests/migration/Migrate16xTo200IT.java | 27 ++
.../resources/migration/snapshots_repository.zip | Bin 709502 -> 880112 bytes
.../actions/MergeProfilesOnPropertyAction.java | 52 +---
.../resources/OSGI-INF/blueprint/blueprint.xml | 1 -
.../services/impl/profiles/ProfileServiceImpl.java | 15 +-
tools/shell-commands/pom.xml | 7 +
.../unomi/shell/migration/MigrationConfig.java | 2 +-
.../unomi/shell/migration/actions/Migrate.java | 1 -
.../unomi/shell/migration/impl/MigrationTo200.java | 286 ---------------------
.../shell/migration/utils/MigrationUtils.java | 47 ++++
...migrate-2.0.0-00-scopesAndProfileAliases.groovy | 20 --
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 70 +++++
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 50 ++++
...roovy => migrate-2.0.0-05-globalReindex.groovy} | 0
...oovy => migrate-2.0.0-10-profileReindex.groovy} | 0
...roovy => migrate-2.0.0-15-eventsReindex.groovy} | 0
.../requestBody/2.0.0/alias_save_bulk.ndjson | 2 +
.../requestBody/2.0.0/profile_merged_delete.json | 7 +
.../requestBody/2.0.0/profile_merged_search.json | 8 +
.../requestBody/2.0.0/scope_save_bulk.ndjson | 2 +
.../{searchScope.json => 2.0.0/scope_search.json} | 0
.../requestBody/bulkSaveProfileAliases.ndjson | 2 -
.../resources/requestBody/bulkSaveScope.ndjson | 2 -
.../requestBody/copyValueScopeToSourceId.json | 6 -
29 files changed, 249 insertions(+), 419 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/Profile.java b/api/src/main/java/org/apache/unomi/api/Profile.java
index 7115bd563..ea9583850 100644
--- a/api/src/main/java/org/apache/unomi/api/Profile.java
+++ b/api/src/main/java/org/apache/unomi/api/Profile.java
@@ -55,6 +55,10 @@ public class Profile extends Item {
private Map<String, Integer> scores;
+ /**
+ * @deprecated since 2.0.0 merge mechanism is now based on profile aliases, and this property is not used anymore
+ */
+ @Deprecated
private String mergedWith;
private Map<String, Consent> consents = new LinkedHashMap<>();
@@ -201,18 +205,17 @@ public class Profile extends Item {
}
/**
- * Retrieves the identifier of the profile this profile is merged with if any.
- *
- * @return the identifier of the profile this profile is merged with if any, {@code null} otherwise
+ * @deprecated since 2.0.0 merge mechanism is now based on profile aliases, and this property is not used anymore
*/
+ @Deprecated
public String getMergedWith() {
return mergedWith;
}
/**
- * TODO: should be removed from the API
- * @param mergedWith new value for mergedWith
+ * @deprecated since 2.0.0 merge mechanism is now based on profile aliases, and this property is not used anymore
*/
+ @Deprecated
public void setMergedWith(String mergedWith) {
this.mergedWith = mergedWith;
}
@@ -290,7 +293,6 @@ public class Profile extends Item {
sb.append(", systemProperties=").append(systemProperties);
sb.append(", segments=").append(segments);
sb.append(", scores=").append(scores);
- sb.append(", mergedWith='").append(mergedWith).append('\'');
sb.append(", consents=").append(consents);
sb.append(", itemId='").append(itemId).append('\'');
sb.append(", itemType='").append(itemType).append('\'');
diff --git a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
index c3aa72aa3..a4af565e2 100644
--- a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
+++ b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
@@ -20,8 +20,9 @@
<details>${project.description}</details>
<feature prerequisite="true" dependency="false">wrap</feature>
<feature dependency="true">unomi-kar</feature>
- <bundle start-level="85">mvn:org.codehaus.groovy/groovy/3.0.3</bundle>
- <bundle start-level="85">mvn:org.codehaus.groovy/groovy-xml/3.0.3</bundle>
+ <bundle start-level="85">mvn:org.codehaus.groovy/groovy/${groovy.version}</bundle>
+ <bundle start-level="85">mvn:org.codehaus.groovy/groovy-xml/${groovy.version}</bundle>
+ <bundle start-level="85">mvn:org.codehaus.groovy/groovy-json/${groovy.version}</bundle>
<bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-services/${project.version}</bundle>
<bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-rest/${project.version}</bundle>
<bundle start-level="85">wrap:mvn:io.github.http-builder-ng/http-builder-ng-core/1.0.4</bundle>
diff --git a/extensions/groovy-actions/services/pom.xml b/extensions/groovy-actions/services/pom.xml
index 770d4da7e..cf747d670 100644
--- a/extensions/groovy-actions/services/pom.xml
+++ b/extensions/groovy-actions/services/pom.xml
@@ -85,6 +85,12 @@
<version>${groovy.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-json</artifactId>
+ <version>${groovy.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index 1db16621a..8cbbea5f2 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -30,39 +30,7 @@ import org.junit.runners.Suite.SuiteClasses;
*/
@RunWith(Suite.class)
@SuiteClasses({
- Migrate16xTo200IT.class,
- BasicIT.class,
- ConditionEvaluatorIT.class,
- ConditionESQueryBuilderIT.class,
- SegmentIT.class,
- ProfileServiceIT.class,
- ProfileImportBasicIT.class,
- ProfileImportSurfersIT.class,
- ProfileImportRankingIT.class,
- ProfileImportActorsIT.class,
- ProfileExportIT.class,
- ProfileMergeIT.class,
- EventServiceIT.class,
- PropertiesUpdateActionIT.class,
- CopyPropertiesActionIT.class,
- IncrementPropertyIT.class,
- InputValidationIT.class,
- ModifyConsentIT.class,
- PatchIT.class,
- ContextServletIT.class,
- SecurityIT.class,
- RuleServiceIT.class,
- PrivacyServiceIT.class,
- GroovyActionsServiceIT.class,
- GraphQLEventIT.class,
- GraphQLListIT.class,
- GraphQLProfileIT.class,
- GraphQLProfilePropertiesIT.class,
- GraphQLSegmentIT.class,
- GraphQLWebSocketIT.class,
- JSONSchemaIT.class,
- GraphQLProfileAliasesIT.class,
- SendEventActionIT.class
+ Migrate16xTo200IT.class
})
public class AllITs {
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
index 8e38424e8..f6e58c048 100644
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
@@ -219,4 +219,6 @@ public class ProfileServiceIT extends BaseIT {
}
}
+ // todo test alias cannot be created on profile itself
+
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
index faf6a87e5..921ae4ab1 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo200IT.java
@@ -28,6 +28,7 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -62,6 +63,7 @@ public class Migrate16xTo200IT extends BaseIT {
@After
public void cleanup() throws InterruptedException {
removeItems(Profile.class);
+ removeItems(ProfileAlias.class);
removeItems(Session.class);
removeItems(Event.class);
removeItems(Scope.class);
@@ -69,6 +71,7 @@ public class Migrate16xTo200IT extends BaseIT {
@Test
public void checkMigratedData() throws Exception {
+ checkMergedProfilesAliases();
checkProfileInterests();
checkScopeHaveBeenCreated();
checkFormEventRestructured();
@@ -82,6 +85,7 @@ public class Migrate16xTo200IT extends BaseIT {
*/
private void checkForMappingUpdates() throws IOException {
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scope/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-profilealias/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-segment/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scoring/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-campaign/_mapping", null).contains("\"entryCondition\":{\"type\":\"object\",\"enabled\":false}"));
@@ -215,4 +219,27 @@ public class Migrate16xTo200IT extends BaseIT {
}
}
}
+
+ /**
+ * Data set contains a master profile: 468ca2bf-7d24-41ea-9ef4-5b96f78207e4
+ * And two profiles that have been merged with this master profile: c33dec90-ffc9-4484-9e61-e42c323f268f and ac5b6b0f-afce-4c4f-9391-4ff0b891b254
+ */
+ private void checkMergedProfilesAliases() {
+ // Check that both profiles aliases have been created and the merged profiles are now deleted.
+ List<String> mergedProfiles = Arrays.asList("c33dec90-ffc9-4484-9e61-e42c323f268f", "ac5b6b0f-afce-4c4f-9391-4ff0b891b254");
+ String masterProfile = "468ca2bf-7d24-41ea-9ef4-5b96f78207e4";
+ for (String mergedProfile : mergedProfiles) {
+ // control the created alias
+ ProfileAlias alias = persistenceService.load(mergedProfile, ProfileAlias.class);
+ Assert.assertNotNull(alias);
+ Assert.assertEquals(alias.getProfileID(), masterProfile);
+
+ // control the merged profile do not exist anymore
+ Assert.assertNull(persistenceService.load(mergedProfile, Profile.class));
+ }
+
+ // Check master profile still exists a no alias have been created for him
+ Assert.assertNotNull(persistenceService.load(masterProfile, Profile.class));
+ Assert.assertNull(persistenceService.load(masterProfile, ProfileAlias.class));
+ }
}
diff --git a/itests/src/test/resources/migration/snapshots_repository.zip b/itests/src/test/resources/migration/snapshots_repository.zip
index e0a81a755..58eb87e72 100644
Binary files a/itests/src/test/resources/migration/snapshots_repository.zip and b/itests/src/test/resources/migration/snapshots_repository.zip differ
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
index 2e21d8b1c..752068594 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
@@ -31,10 +31,6 @@ import org.apache.unomi.persistence.spi.PersistenceService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@@ -46,13 +42,9 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
private EventService eventService;
private DefinitionsService definitionsService;
private PrivacyService privacyService;
- private ConfigSharingService configSharingService;
private int maxProfilesInOneMerge = -1;
public int execute(Action action, Event event) {
-// String profileIdCookieName = (String) configSharingService.getProperty("profileIdCookieName");
-// String profileIdCookieDomain = (String) configSharingService.getProperty("profileIdCookieDomain");
-// Integer profileIdCookieMaxAgeInSeconds = (Integer) configSharingService.getProperty("profileIdCookieMaxAgeInSeconds");
Profile profile = event.getProfile();
if (profile instanceof Persona || profile.isAnonymousProfile()) {
@@ -84,15 +76,7 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
propertyCondition.setParameter("propertyName", "systemProperties." + mergeProfilePropertyName);
propertyCondition.setParameter("propertyValue", mergeProfilePropertyValue);
- Condition excludeMergedProfilesCondition = new Condition(definitionsService.getConditionType("profilePropertyCondition"));
- excludeMergedProfilesCondition.setParameter("comparisonOperator", "missing");
- excludeMergedProfilesCondition.setParameter("propertyName", "mergedWith");
-
- Condition c = new Condition(definitionsService.getConditionType("booleanCondition"));
- c.setParameter("operator", "and");
- c.setParameter("subConditions", Arrays.asList(propertyCondition, excludeMergedProfilesCondition));
-
- final List<Profile> profiles = persistenceService.query(c, "properties.firstVisit", Profile.class, 0, maxProfilesInOneMerge).getList();
+ final List<Profile> profiles = persistenceService.query(propertyCondition, "properties.firstVisit", Profile.class, 0, maxProfilesInOneMerge).getList();
// Check if the user switched to another profile
if (StringUtils.isNotEmpty(mergeProfilePreviousPropertyValue) && !mergeProfilePreviousPropertyValue.equals(mergeProfilePropertyValue)) {
@@ -111,14 +95,6 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
}
logger.info("Different users, switch to " + profile.getItemId());
-
-// HttpServletResponse httpServletResponse = (HttpServletResponse) event.getAttributes().get(Event.HTTP_RESPONSE_ATTRIBUTE);
-// HttpServletRequest httpServletRequest = (HttpServletRequest) event.getAttributes().get(Event.HTTP_REQUEST_ATTRIBUTE);
-// if (httpServletRequest != null) {
-// sendProfileCookie(profile, httpServletResponse, profileIdCookieName, profileIdCookieDomain,
-// profileIdCookieMaxAgeInSeconds, profileIdCookieHttpOnly, httpServletRequest.isSecure());
-// }
-
// At the end of the merge, we must set the merged profile as profile event to process other Actions
event.setProfileId(profile.getItemId());
event.setProfile(profile);
@@ -157,13 +133,6 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
// Profile has changed
if (forceEventProfileAsMaster || !masterProfile.getItemId().equals(profileId)) {
- HttpServletResponse httpServletResponse = (HttpServletResponse) event.getAttributes().get(Event.HTTP_RESPONSE_ATTRIBUTE);
- // we still send back the current profile cookie. It will be changed on the next request to the ContextServlet.
- // The current profile will be deleted only then because we cannot delete it right now (too soon)
-// if (httpServletRequest != null) {
-// sendProfileCookie(profile, httpServletResponse, profileIdCookieName, profileIdCookieDomain,
-// profileIdCookieMaxAgeInSeconds, profileIdCookieHttpOnly, httpServletRequest.isSecure());
-// }
final String masterProfileId = masterProfile.getItemId();
// At the end of the merge, we must set the merged profile as profile event to process other Actions
@@ -242,20 +211,6 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
}
}
-// private static void sendProfileCookie(Profile profile, ServletResponse response, String profileIdCookieName, String profileIdCookieDomain, int cookieAgeInSeconds) {
-// if (response != null && response instanceof HttpServletResponse) {
-// HttpServletResponse httpServletResponse = (HttpServletResponse) response;
-// if (!(profile instanceof Persona)) {
-// httpServletResponse.addHeader("Set-Cookie",
-// profileIdCookieName + "=" + profile.getItemId() +
-// "; Path=/" +
-// "; Max-Age=" + cookieAgeInSeconds +
-// (StringUtils.isNotBlank(profileIdCookieDomain) ? ("; Domain=" + profileIdCookieDomain) : "") +
-// "; SameSite=Lax");
-// }
-// }
-// }
-
public void setProfileService(ProfileService profileService) {
this.profileService = profileService;
}
@@ -276,12 +231,7 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
this.definitionsService = definitionsService;
}
- public void setConfigSharingService(ConfigSharingService configSharingService) {
- this.configSharingService = configSharingService;
- }
-
public void setMaxProfilesInOneMerge(String maxProfilesInOneMerge) {
this.maxProfilesInOneMerge = Integer.parseInt(maxProfilesInOneMerge);
}
-
}
diff --git a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index fd9512a27..3222eb680 100644
--- a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -315,7 +315,6 @@
<property name="persistenceService" ref="persistenceService"/>
<property name="definitionsService" ref="definitionsService"/>
<property name="privacyService" ref="privacyService"/>
- <property name="configSharingService" ref="configSharingService"/>
<property name="maxProfilesInOneMerge" value="${base.maxProfilesInOneMerge}"/>
</bean>
</service>
diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
index cdfa8c8b3..a11189ae9 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
@@ -563,7 +563,14 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
@Override
public void addAliasToProfile(String profileID, String alias, String clientID) {
+ if (Objects.equals(alias, profileID)) {
+ throw new IllegalArgumentException("Alias cannot be created on itself, please use an alias different from the profile ID");
+ }
+
ProfileAlias profileAlias = persistenceService.load(alias, ProfileAlias.class);
+ if (profileAlias != null && !Objects.equals(profileAlias.getProfileID(), profileID)) {
+ throw new IllegalArgumentException("Alias \"" + alias + "\" already used by profile with ID = \"" + profileAlias.getProfileID() + "\"");
+ }
if (profileAlias == null) {
profileAlias = new ProfileAlias();
@@ -578,8 +585,6 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
profileAlias.setModifiedTime(creationTime);
persistenceService.save(profileAlias);
- } else if (!Objects.equals(profileAlias.getProfileID(), profileID)) {
- throw new IllegalArgumentException("Alias \"" + alias + "\" already used by profile with ID = \"" + profileID + "\"");
}
}
@@ -678,12 +683,6 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
if (persona) {
persistenceService.remove(profileId, Persona.class);
} else {
- Condition mergeCondition = new Condition(definitionsService.getConditionType("profilePropertyCondition"));
- mergeCondition.setParameter("propertyName", "mergedWith");
- mergeCondition.setParameter("comparisonOperator", "equals");
- mergeCondition.setParameter("propertyValue", profileId);
- persistenceService.removeByQuery(mergeCondition, Profile.class);
-
Condition removeAliasesCondition = new Condition(definitionsService.getConditionType("profileAliasesPropertyCondition"));
removeAliasesCondition.setParameter("propertyName", "profileID");
removeAliasesCondition.setParameter("comparisonOperator", "equals");
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index e3e863d0a..1b73ab209 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -86,6 +86,13 @@
<version>${groovy.version}</version>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-json</artifactId>
+ <version>${groovy.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
index da32faba2..41f37aac7 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationConfig.java
@@ -53,7 +53,7 @@ public class MigrationConfig {
m.put(CONFIG_ES_ADDRESS, new MigrationConfigProperty("Enter ElasticSearch TARGET address (default: http://localhost:9200): ", "http://localhost:9200"));
m.put(CONFIG_ES_LOGIN, new MigrationConfigProperty("Enter ElasticSearch TARGET login (default: none): ", ""));
m.put(CONFIG_ES_PASSWORD, new MigrationConfigProperty("Enter ElasticSearch TARGET password (default: none): ", ""));
- m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ?", null));
+ m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ? (yes/no)", null));
m.put(INDEX_PREFIX, new MigrationConfigProperty("Enter ElasticSearch Unomi indices prefix (default: context): ", "context"));
m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 3): ", "3"));
m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
index 585bd20c6..ea5afdfde 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java
@@ -54,7 +54,6 @@ import static org.apache.unomi.shell.migration.MigrationConfig.*;
@Service
public class Migrate implements Action {
-
@Reference
Session session;
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
deleted file mode 100644
index b1de1815d..000000000
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.unomi.shell.migration.impl;
-
-import org.apache.http.HttpStatus;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-import org.apache.karaf.shell.api.console.Session;
-import org.apache.unomi.shell.migration.Migration;
-import org.apache.unomi.shell.migration.MigrationConfig;
-import org.apache.unomi.shell.migration.utils.ConsoleUtils;
-import org.apache.unomi.shell.migration.utils.MigrationUtils;
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.osgi.framework.BundleContext;
-
-import java.io.IOException;
-import java.net.URI;
-import java.time.Instant;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-public class MigrationTo200 implements Migration {
-
- private CloseableHttpClient httpClient;
- private Session session;
- private String esAddress;
- private String indexPrefix;
- private BundleContext bundleContext;
- private MigrationConfig migrationConfig;
-
- @Override
- public void execute(Session session, CloseableHttpClient httpClient, MigrationConfig migrationConfig, BundleContext bundleContext) throws IOException {
- this.httpClient = httpClient;
- this.session = session;
- this.esAddress = migrationConfig.getString(MigrationConfig.CONFIG_ES_ADDRESS, session);
- this.indexPrefix = migrationConfig.getString(MigrationConfig.INDEX_PREFIX, session);
- this.bundleContext = bundleContext;
- this.migrationConfig = migrationConfig;
-
- doExecute();
- }
-
- private void doExecute() throws IOException {
- Set<String> indexes = MigrationUtils.getIndexesPrefixedBy(httpClient, esAddress, indexPrefix + "-event-");
- createScopeMapping();
- createScopes(getSetOfScopes(indexes));
- createProfileAliasDocumentsFromProfile();
- }
-
- private boolean scopeIndexNotExists() throws IOException {
- final HttpGet httpGet = new HttpGet(esAddress + "/" + indexPrefix + "-scope");
-
- httpGet.addHeader("Accept", "application/json");
- httpGet.addHeader("Content-Type", "application/json");
-
- try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
- return response.getStatusLine().getStatusCode() != HttpStatus.SC_OK;
- }
- }
-
- private void createScopeMapping() throws IOException {
-
- if (scopeIndexNotExists()) {
- System.out.println("Creation for index = \"" + indexPrefix + "-scope\" starting.");
- final HttpPut httpPost = new HttpPut(esAddress + "/" + indexPrefix + "-scope");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/json");
-
- String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json");
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json");
- // We intentionally extract setting from profile index, because the scope index doesnt exist yet, and all indices share the same configuration regarding shards, replicas, etc ..
- String request = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, indexPrefix + "-profile", mapping);
-
- httpPost.setEntity(new StringEntity(request));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println(indexPrefix + "-scope has been correctly created");
- } else {
- System.out.println(
- "Failed to create the index " + indexPrefix + "-scope.Code:" + response.getStatusLine().getStatusCode());
- throw new RuntimeException("Can not create the scope index. Stop the execution of the migration.");
- }
- }
- } else {
- System.out.println("The scope index already exists. Skipping the creation of this index");
- }
-
- }
-
- private void createScopes(Set<String> scopes) throws IOException {
- final StringBuilder body = new StringBuilder();
- String saveScopeBody = MigrationUtils.resourceAsString(bundleContext,"requestBody/bulkSaveScope.ndjson");
- scopes.forEach(scope -> body.append(saveScopeBody.replace("$scope", scope)));
-
- final HttpPost httpPost = new HttpPost(esAddress + "/" + indexPrefix + "-scope/_bulk");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/x-ndjson");
-
- httpPost.setEntity(new StringEntity(body.toString()));
-
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope successfully finished");
- } else {
- System.out.println("Creating the \"scopes\" into the index " + indexPrefix + "-scope has failed" + response.getStatusLine()
- .getStatusCode());
- }
- }
- }
-
- private Set<String> getSetOfScopes(Set<String> indices) throws IOException {
- String joinedIndices = String.join(",", indices);
- final HttpPost httpPost = new HttpPost(esAddress + "/" + joinedIndices + "/_search");
-
- httpPost.addHeader("Accept", "application/json");
- httpPost.addHeader("Content-Type", "application/json");
-
- String request = MigrationUtils.resourceAsString(bundleContext,"requestBody/searchScope.json");
-
- httpPost.setEntity(new StringEntity(request));
-
- Set<String> scopes = new HashSet<>();
- try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
- JSONObject responseAsJson = new JSONObject(EntityUtils.toString(response.getEntity()));
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- System.out.println("Getting the \"scope\" values from the events successfully finished. " + "Number of scope to create: "
- + responseAsJson.getJSONObject("aggregations").getJSONObject("bucketInfos").get("count").toString());
- scopes = StreamSupport
- .stream(responseAsJson.getJSONObject("aggregations").getJSONObject("scopes").getJSONArray("buckets").spliterator(),
- false).map(bucketElement -> ((JSONObject) bucketElement).getString("key")).collect(Collectors.toSet());
- } else {
- System.out.println(
- "Getting the \"scope\" values from the event has failed. Code: " + response.getStatusLine().getStatusCode());
- }
- }
- return scopes;
- }
-
- private void createProfileAliasDocumentsFromProfile() throws IOException {
- System.out.println("Migration \"Create profileAlias from profile\" started");
- Instant migrationTime = Instant.now();
- int size = 1000;
- doProcessProfiles(migrationTime, size);
- System.out.println("Migration \"Create profileAlias from profile\" completed.");
- }
-
- private void doProcessProfiles(Instant migrationTime, int offset) throws IOException {
- CloseableHttpResponse response = null;
- try {
- response = httpClient.execute(createSearchRequest(offset));
-
- while (true) {
- JSONObject responseAsJson = getResponseAsJSON(response);
-
- String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id"): null;
- JSONArray hits = getProfileHits(responseAsJson);
-
- if (hits.length() == 0) {
- if (scrollId != null) {
- CloseableHttpResponse deleteScrollResponse = httpClient.execute(createDeleteScrollRequest(scrollId));
- if (deleteScrollResponse != null) {
- deleteScrollResponse.close();
- }
- }
- break;
- }
-
- StringBuilder bulkCreateRequest = new StringBuilder();
- for (Object o : hits) {
- JSONObject hit = (JSONObject) o;
- if (hit.has("_source")) {
- JSONObject profile = hit.getJSONObject("_source");
- if (profile.has("itemId")) {
- String itemId = profile.getString("itemId");
- String bulkSaveProfileAliases = MigrationUtils.resourceAsString(bundleContext,"requestBody/bulkSaveProfileAliases.ndjson");
- bulkCreateRequest.append(bulkSaveProfileAliases.
- replace("$itemId", itemId).
- replace("$migrationTime", migrationTime.toString()));
- }
- }
- }
-
- CloseableHttpResponse bulkResponse = httpClient.execute(createProfileAliasRequest(bulkCreateRequest.toString()));
- if (bulkResponse != null) {
- bulkResponse.close();
- }
-
- response = httpClient.execute(createSearchRequestWithScrollId(scrollId));
- }
- } finally {
- if (response != null) {
- response.close();
- }
- }
- }
-
- private JSONObject getResponseAsJSON(CloseableHttpResponse response) throws IOException {
- if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
- return new JSONObject(EntityUtils.toString(response.getEntity()));
- }
- return new JSONObject();
- }
-
- private JSONArray getProfileHits(JSONObject responseAsJson) {
- if (responseAsJson.has("hits")) {
- JSONObject hitsObject = responseAsJson.getJSONObject("hits");
- if (hitsObject.has("hits")) {
- return hitsObject.getJSONArray("hits");
- }
- }
- return new JSONArray();
- }
-
- private HttpPost createSearchRequestWithScrollId(final String scrollId) throws IOException {
- final String requestBody = "{\n" +
- " \"scroll_id\": \"" + scrollId + "\",\n" +
- " \"scroll\": \"1h\"\n" +
- "}";
-
- final HttpPost request = new HttpPost(esAddress + "/_search/scroll");
-
- request.addHeader("Accept", "application/json");
- request.addHeader("Content-Type", "application/json");
- request.setEntity(new StringEntity(requestBody));
-
- return request;
- }
-
- private HttpGet createSearchRequest(int size) {
- return new HttpGet(esAddress + "/context-profile/_search?&scroll=1h&_source_includes=itemId&size=" + size);
- }
-
- private HttpEntityEnclosingRequestBase createDeleteScrollRequest(final String scrollId) throws IOException {
- final HttpEntityEnclosingRequestBase deleteRequest = new HttpEntityEnclosingRequestBase() {
- @Override
- public String getMethod() {
- return "DELETE";
- }
- };
-
- deleteRequest.setURI(URI.create(esAddress + "/_search/scroll"));
- deleteRequest.setEntity(new StringEntity("{ \"scroll_id\": \"" + scrollId + "\" }"));
- deleteRequest.addHeader("Accept", "application/json");
- deleteRequest.addHeader("Content-Type", "application/json");
-
- return deleteRequest;
- }
-
- private HttpPost createProfileAliasRequest(String bulkRequestAsString) throws IOException {
- final HttpPost bulkRequest = new HttpPost(esAddress + "/context-profilealias/_bulk");
-
- bulkRequest.addHeader("Accept", "application/json");
- bulkRequest.addHeader("Content-Type", "application/json");
- bulkRequest.setEntity(new StringEntity(bulkRequestAsString));
-
- return bulkRequest;
- }
-}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index e93998b50..6297a8951 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -23,6 +23,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
+import org.json.JSONArray;
import org.json.JSONObject;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
@@ -87,6 +88,13 @@ public class MigrationUtils {
}
}
+ public static boolean indexExists(CloseableHttpClient httpClient, String esAddress, String indexName) throws IOException {
+ final HttpGet httpGet = new HttpGet(esAddress + "/" + indexName);
+ try (CloseableHttpResponse response = httpClient.execute(httpGet)) {
+ return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK;
+ }
+ }
+
public static Set<String> getIndexesPrefixedBy(CloseableHttpClient httpClient, String esAddress, String prefix) throws IOException {
try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
@@ -156,6 +164,45 @@ public class MigrationUtils {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
}
+ public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
+ String response = HttpUtils.executePostRequest(httpClient, esAddress + queryURL + "?scroll=" + scrollDuration, query, null);
+
+ while (true) {
+ JSONObject responseAsJson = new JSONObject(response);
+ String scrollId = responseAsJson.has("_scroll_id") ? responseAsJson.getString("_scroll_id"): null;
+ JSONArray hits = new JSONArray();
+ if (responseAsJson.has("hits")) {
+ JSONObject hitsObject = responseAsJson.getJSONObject("hits");
+ if (hitsObject.has("hits")) {
+ hits = hitsObject.getJSONArray("hits");
+ }
+ }
+
+ // no more results, delete scroll
+ if (hits.length() == 0) {
+ if (scrollId != null) {
+ HttpUtils.executeDeleteRequest(httpClient, esAddress + "/_search/scroll/" + scrollId, null);
+ }
+ break;
+ }
+
+ // execute callback
+ if (scrollCallback != null) {
+ scrollCallback.execute(hits.toString());
+ }
+
+ // scroll
+ response = HttpUtils.executePostRequest(httpClient, esAddress + "/_search/scroll", "{\n" +
+ " \"scroll_id\": \"" + scrollId + "\",\n" +
+ " \"scroll\": \"" + scrollDuration + "\"\n" +
+ "}", null);
+ }
+ }
+
+ public interface ScrollCallback {
+ void execute(String hits);
+ }
+
private static String getScriptPart(String painlessScript) {
return ", \"script\": {\"source\": \"" + painlessScript + "\", \"lang\": \"painless\"}";
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-00-scopesAndProfileAliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-00-scopesAndProfileAliases.groovy
deleted file mode 100644
index 80d66d903..000000000
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-00-scopesAndProfileAliases.groovy
+++ /dev/null
@@ -1,20 +0,0 @@
-import org.apache.unomi.shell.migration.impl.MigrationTo200
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-new MigrationTo200().execute(session, httpClient, migrationConfig, bundleContext)
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
new file mode 100644
index 000000000..6ad9e9b8f
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -0,0 +1,70 @@
+import groovy.json.JsonSlurper
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+import java.time.Instant
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+Instant migrationTime = Instant.now();
+def jsonSlurper = new JsonSlurper()
+String aliasSaveBulkRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/alias_save_bulk.ndjson");
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String aliasIndex = indexPrefix + "-profilealias"
+String profileIndex = indexPrefix + "-profile"
+
+// create alias index
+if (!MigrationUtils.indexExists(httpClient, esAddress, aliasIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, profileIndex, mapping)
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + aliasIndex, newIndexSettings, null)
+
+ // scroll search on profiles merged
+ String profileMergedSearchRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_search.json")
+ MigrationUtils.scrollQuery(httpClient, esAddress, "/" + profileIndex + "/_search", profileMergedSearchRequest, "1h", new MigrationUtils.ScrollCallback() {
+ @Override
+ void execute(String hits) {
+ // create aliases for those merged profiles and delete them.
+ def jsonHits = jsonSlurper.parseText(hits)
+ ConsoleUtils.printMessage(session, "Detected: " + jsonHits.size() + " profile alias to create")
+ final StringBuilder bulkSaveRequest = new StringBuilder()
+ jsonHits.each {
+ jsonHit -> {
+ // check that master still exists before creating alias:
+ def masterProfile = jsonSlurper.parseText(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + profileIndex + "/_doc/" + jsonHit._source.mergedWith, null))
+ if (masterProfile.found) {
+ bulkSaveRequest.append(aliasSaveBulkRequest
+ .replace("##itemId##", jsonHit._source.itemId)
+ .replace("##profileId##", jsonHit._source.mergedWith)
+ .replace("##migrationTime##", migrationTime.toString()))
+ }
+ }
+ }
+ if (bulkSaveRequest.length() > 0) {
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + aliasIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ }
+ }
+ })
+
+ // delete existing merged profiles
+ String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
new file mode 100644
index 000000000..771c3ec70
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -0,0 +1,50 @@
+import groovy.json.JsonSlurper
+import org.apache.unomi.shell.migration.utils.ConsoleUtils
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+def jsonSlurper = new JsonSlurper()
+String searchScopesRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/scope_search.json")
+String saveScopeRequestBulk = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scope_save_bulk.ndjson")
+String esAddress = migrationConfig.getString("esAddress", session)
+String indexPrefix = migrationConfig.getString("indexPrefix", session)
+String scopeIndex = indexPrefix + "-scope"
+
+// Create scope index:
+if (!MigrationUtils.indexExists(httpClient, esAddress, scopeIndex)) {
+ String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(httpClient, esAddress, baseRequest, indexPrefix + "-profile", mapping)
+ HttpUtils.executePutRequest(httpClient, esAddress + "/" + scopeIndex, newIndexSettings, null)
+
+ // search existing scopes from event
+ def searchResponse = jsonSlurper.parseText(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexPrefix + "-event-*/_search", searchScopesRequest, null))
+ ConsoleUtils.printMessage(session, "Detected: " + searchResponse.aggregations.bucketInfos.count + " scopes to create")
+
+ // create scopes
+ def buckets = searchResponse.aggregations.scopes.buckets
+ if (buckets != null && buckets.size() > 0) {
+ final StringBuilder bulkSaveRequest = new StringBuilder()
+ buckets.each {
+ bucket -> bulkSaveRequest.append(saveScopeRequestBulk.replace("##scope##", bucket.key))
+ }
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + scopeIndex + "/_bulk", bulkSaveRequest.toString(), null)
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-globalReindex.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-03-profileReindex.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
similarity index 100%
rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-04-eventsReindex.groovy
rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/alias_save_bulk.ndjson b/tools/shell-commands/src/main/resources/requestBody/2.0.0/alias_save_bulk.ndjson
new file mode 100644
index 000000000..f69e1b927
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/alias_save_bulk.ndjson
@@ -0,0 +1,2 @@
+{ "create" : { "_id": "##itemId##" }}
+{ "itemId": "##itemId##", "itemType": "profileAlias", "profileID": "##profileId##", "scope": null, "clientID": "defaultClientId", "creationTime": "##migrationTime##", "modifiedTime": "##migrationTime##"}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_delete.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_delete.json
new file mode 100644
index 000000000..a6d7cb726
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_delete.json
@@ -0,0 +1,7 @@
+{
+ "query": {
+ "exists": {
+ "field": "mergedWith"
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_search.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_search.json
new file mode 100644
index 000000000..d6598b39f
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/profile_merged_search.json
@@ -0,0 +1,8 @@
+{
+ "size": 1000,
+ "query": {
+ "exists": {
+ "field": "mergedWith"
+ }
+ }
+}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_save_bulk.ndjson b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_save_bulk.ndjson
new file mode 100644
index 000000000..9785285f7
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_save_bulk.ndjson
@@ -0,0 +1,2 @@
+{"create": {"_id": "##scope##"}}
+{"itemId": "##scope##", "itemType": "scope", "metadata": {"id": "##scope##" }}
diff --git a/tools/shell-commands/src/main/resources/requestBody/searchScope.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_search.json
similarity index 100%
rename from tools/shell-commands/src/main/resources/requestBody/searchScope.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/scope_search.json
diff --git a/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson b/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
deleted file mode 100644
index ad886ca35..000000000
--- a/tools/shell-commands/src/main/resources/requestBody/bulkSaveProfileAliases.ndjson
+++ /dev/null
@@ -1,2 +0,0 @@
-{ "create" : { "_id": "$itemId" }}
-{ "itemId": "$itemId", "itemType": "profileAlias", "profileID": "$itemId", "scope": null, "clientID": "defaultClientId", "creationTime": "$migrationTime", "modifiedTime": "$migrationTime"}
diff --git a/tools/shell-commands/src/main/resources/requestBody/bulkSaveScope.ndjson b/tools/shell-commands/src/main/resources/requestBody/bulkSaveScope.ndjson
deleted file mode 100644
index 32f9bb0bd..000000000
--- a/tools/shell-commands/src/main/resources/requestBody/bulkSaveScope.ndjson
+++ /dev/null
@@ -1,2 +0,0 @@
-{"index": {"_id": "$scope"}}
-{"itemId": "$scope", "itemType": "scope", "metadata": {"id": "$scope" }}
diff --git a/tools/shell-commands/src/main/resources/requestBody/copyValueScopeToSourceId.json b/tools/shell-commands/src/main/resources/requestBody/copyValueScopeToSourceId.json
deleted file mode 100644
index 8d9a1ddaa..000000000
--- a/tools/shell-commands/src/main/resources/requestBody/copyValueScopeToSourceId.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "script": {
- "source": "ctx._source.sourceId = ctx._source.scope",
- "lang": "painless"
- }
-}