You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/18 14:52:21 UTC
[unomi] branch master updated: UNOMI-640: safer way of retreiving index config in migration (#476)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 7130ffa69 UNOMI-640: safer way of retreiving index config in migration (#476)
7130ffa69 is described below
commit 7130ffa69ddbdd1649d28845e2fe062f1c3a7d07
Author: kevan Jahanshahi <ke...@jahia.com>
AuthorDate: Thu Aug 18 16:52:15 2022 +0200
UNOMI-640: safer way of retreiving index config in migration (#476)
---
.../shell/migration/service/MigrationConfig.java | 10 +++++++++-
.../shell/migration/utils/MigrationUtils.java | 22 ++++++++--------------
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 3 ++-
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 2 +-
.../migrate-2.0.0-05-globalReindex.groovy | 2 +-
.../migrate-2.0.0-10-profileReindex.groovy | 2 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 4 +---
.../main/resources/org.apache.unomi.migration.cfg | 8 ++++++++
8 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
index 33f7f321b..16fb6728a 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
@@ -41,6 +41,10 @@ public class MigrationConfig {
public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
public static final String TOTAL_FIELDS_LIMIT = "mapping.total_fields.limit";
public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
+ public static final String MONTHLY_NUMBER_OF_SHARDS = "monthlyIndex." + NUMBER_OF_SHARDS;
+ public static final String MONTHLY_NUMBER_OF_REPLICAS = "monthlyIndex." + NUMBER_OF_REPLICAS;
+ public static final String MONTHLY_TOTAL_FIELDS_LIMIT = "monthlyIndex." + TOTAL_FIELDS_LIMIT;
+ public static final String MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH = "monthlyIndex." + MAX_DOC_VALUE_FIELDS_SEARCH;
public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
protected static final Map<String, MigrationConfigProperty> configProperties;
@@ -52,10 +56,14 @@ public class MigrationConfig {
m.put(CONFIG_ES_PASSWORD, new MigrationConfigProperty("Enter ElasticSearch TARGET password (default: none): ", ""));
m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ? (yes/no)", null));
m.put(INDEX_PREFIX, new MigrationConfigProperty("Enter ElasticSearch Unomi indices prefix (default: context): ", "context"));
- m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 3): ", "3"));
+ m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 5): ", "5"));
m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
m.put(TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
m.put(MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
+ m.put(MONTHLY_NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: number_of_shards (default: 5): ", "5"));
+ m.put(MONTHLY_NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: number_of_replicas (default: 0): ", "0"));
+ m.put(MONTHLY_TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
+ m.put(MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
m.put(MIGRATION_HISTORY_RECOVER, new MigrationConfigProperty("We found an existing migration attempt, should we restart from it ? (this will avoid redoing steps already completed successfully) (yes/no)", null));
configProperties = Collections.unmodifiableMap(m);
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index d44be389c..a57d79dec 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -41,6 +41,8 @@ import java.util.Enumeration;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
+
/**
* @author dgaillard
*/
@@ -123,20 +125,12 @@ public class MigrationUtils {
throw new RuntimeException("no mapping found in bundles for: " + fileName);
}
- public static String buildIndexCreationRequest(CloseableHttpClient httpClient, String esAddress, String baseIndexSettings,
- String originalIndexForSettingsExtraction, String mapping) throws IOException {
-
- String settings = baseIndexSettings;
-
- // Extract existing settings on index that still exists
- if (originalIndexForSettingsExtraction != null) {
- JSONObject originalIndexSettings = new JSONObject(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + originalIndexForSettingsExtraction + "/_settings", null));
- settings = settings
- .replace("#numberOfShards", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getString("number_of_shards"))
- .replace("#numberOfReplicas", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getString("number_of_replicas"))
- .replace("#maxDocValueFieldsSearch", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getString("max_docvalue_fields_search"))
- .replace("#mappingTotalFieldsLimit", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getJSONObject("mapping").getJSONObject("total_fields").getString("limit"));
- }
+ public static String buildIndexCreationRequest(String baseIndexSettings, String mapping, MigrationContext context, boolean isMonthlyIndex) throws IOException {
+ String settings = baseIndexSettings
+ .replace("#numberOfShards", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_SHARDS : NUMBER_OF_SHARDS))
+ .replace("#numberOfReplicas", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_REPLICAS : NUMBER_OF_REPLICAS))
+ .replace("#maxDocValueFieldsSearch", context.getConfigString(isMonthlyIndex ? MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH : MAX_DOC_VALUE_FIELDS_SEARCH))
+ .replace("#mappingTotalFieldsLimit", context.getConfigString(isMonthlyIndex ? MONTHLY_TOTAL_FIELDS_LIMIT : TOTAL_FIELDS_LIMIT));
return settings.replace("#mappings", mapping);
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index dcc2a0511..685d44288 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -1,4 +1,5 @@
import groovy.json.JsonSlurper
+import org.apache.unomi.shell.migration.service.MigrationConfig
import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
@@ -36,7 +37,7 @@ context.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, aliasIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, profileIndex, mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseRequest, mapping, context, false)
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + aliasIndex, newIndexSettings, null)
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 3f46f0dd6..d9aa5f889 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -33,7 +33,7 @@ context.performMigrationStep("2.0.0-create-scope-index", () -> {
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, "${indexPrefix}-profile", mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseRequest, mapping, context, false)
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null)
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 826243ae4..dd841ff11 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -26,6 +26,6 @@ String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBod
String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"];
indicesToReindex.each { indexToReindex ->
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "${indexToReindex}.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-profile", mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReindex.toLowerCase()}", newIndexSettings, null, context)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index 4711056c0..bcb4e6ae7 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -24,6 +24,6 @@ String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-segment", mapping)
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, indexPrefix + "-profile",
newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), context)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index 941ff82a8..a9dc7ae1f 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -32,10 +32,8 @@ context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () ->
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, true)
Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-")
-// use session indices to extract monthly index settings
-Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-session-")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, sessionIndices[0], mapping)
eventIndices.each { eventIndex ->
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context)
}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index 5749e4cf9..f4cd52771 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -24,6 +24,14 @@ esLogin = ${org.apache.unomi.elasticsearch.username:-}
esPassword = ${org.apache.unomi.elasticsearch.password:-}
httpClient.trustAllCertificates = ${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false}
indexPrefix = ${org.apache.unomi.elasticsearch.index.prefix:-context}
+monthlyIndex.number_of_shards=${org.apache.unomi.elasticsearch.monthlyIndex.nbShards:-5}
+monthlyIndex.number_of_replicas=${org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas:-0}
+monthlyIndex.mapping.total_fields.limit=${org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000}
+monthlyIndex.max_docvalue_fields_search=${org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch:-1000}
+number_of_shards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5}
+number_of_replicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0}
+mapping.total_fields.limit=${org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit:-1000}
+max_docvalue_fields_search=${org.apache.unomi.elasticsearch.defaultIndex.indexMaxDocValueFieldsSearch:-1000}
# Should the migration try to recover from a previous run ?
# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)