You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2022/08/18 14:08:26 UTC
[unomi] branch improveIndexConfigRetrievalInMigration created (now 1dd04237f)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a change to branch improveIndexConfigRetrievalInMigration
in repository https://gitbox.apache.org/repos/asf/unomi.git
at 1dd04237f UNOMI-640: safer way of retreiving index config in migration
This branch includes the following new commits:
new 1dd04237f UNOMI-640: safer way of retreiving index config in migration
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-640: safer way of retreiving index config in migration
Posted by jk...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch improveIndexConfigRetrievalInMigration
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 1dd04237fa00e668269a271a2fd918f66dd52763
Author: Kevan <ke...@jahia.com>
AuthorDate: Thu Aug 18 16:07:56 2022 +0200
UNOMI-640: safer way of retreiving index config in migration
---
.../shell/migration/service/MigrationConfig.java | 10 +++++++++-
.../shell/migration/utils/MigrationUtils.java | 22 ++++++++--------------
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 3 ++-
.../cxs/migration/migrate-2.0.0-02-scopes.groovy | 2 +-
.../migrate-2.0.0-05-globalReindex.groovy | 2 +-
.../migrate-2.0.0-10-profileReindex.groovy | 2 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 4 +---
.../main/resources/org.apache.unomi.migration.cfg | 8 ++++++++
8 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
index 33f7f321b..16fb6728a 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java
@@ -41,6 +41,10 @@ public class MigrationConfig {
public static final String NUMBER_OF_REPLICAS = "number_of_replicas";
public static final String TOTAL_FIELDS_LIMIT = "mapping.total_fields.limit";
public static final String MAX_DOC_VALUE_FIELDS_SEARCH = "max_docvalue_fields_search";
+ public static final String MONTHLY_NUMBER_OF_SHARDS = "monthlyIndex." + NUMBER_OF_SHARDS;
+ public static final String MONTHLY_NUMBER_OF_REPLICAS = "monthlyIndex." + NUMBER_OF_REPLICAS;
+ public static final String MONTHLY_TOTAL_FIELDS_LIMIT = "monthlyIndex." + TOTAL_FIELDS_LIMIT;
+ public static final String MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH = "monthlyIndex." + MAX_DOC_VALUE_FIELDS_SEARCH;
public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory";
protected static final Map<String, MigrationConfigProperty> configProperties;
@@ -52,10 +56,14 @@ public class MigrationConfig {
m.put(CONFIG_ES_PASSWORD, new MigrationConfigProperty("Enter ElasticSearch TARGET password (default: none): ", ""));
m.put(CONFIG_TRUST_ALL_CERTIFICATES, new MigrationConfigProperty("We need to initialize a HttpClient, do we need to trust all certificates ? (yes/no)", null));
m.put(INDEX_PREFIX, new MigrationConfigProperty("Enter ElasticSearch Unomi indices prefix (default: context): ", "context"));
- m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 3): ", "3"));
+ m.put(NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_shards (default: 5): ", "5"));
m.put(NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: number_of_replicas (default: 0): ", "0"));
m.put(TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
m.put(MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch index mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
+ m.put(MONTHLY_NUMBER_OF_SHARDS, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: number_of_shards (default: 5): ", "5"));
+ m.put(MONTHLY_NUMBER_OF_REPLICAS, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: number_of_replicas (default: 0): ", "0"));
+ m.put(MONTHLY_TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000"));
+ m.put(MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000"));
m.put(MIGRATION_HISTORY_RECOVER, new MigrationConfigProperty("We found an existing migration attempt, should we restart from it ? (this will avoid redoing steps already completed successfully) (yes/no)", null));
configProperties = Collections.unmodifiableMap(m);
}
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index d44be389c..a57d79dec 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -41,6 +41,8 @@ import java.util.Enumeration;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.unomi.shell.migration.service.MigrationConfig.*;
+
/**
* @author dgaillard
*/
@@ -123,20 +125,12 @@ public class MigrationUtils {
throw new RuntimeException("no mapping found in bundles for: " + fileName);
}
- public static String buildIndexCreationRequest(CloseableHttpClient httpClient, String esAddress, String baseIndexSettings,
- String originalIndexForSettingsExtraction, String mapping) throws IOException {
-
- String settings = baseIndexSettings;
-
- // Extract existing settings on index that still exists
- if (originalIndexForSettingsExtraction != null) {
- JSONObject originalIndexSettings = new JSONObject(HttpUtils.executeGetRequest(httpClient, esAddress + "/" + originalIndexForSettingsExtraction + "/_settings", null));
- settings = settings
- .replace("#numberOfShards", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getString("number_of_shards"))
- .replace("#numberOfReplicas", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getString("number_of_replicas"))
- .replace("#maxDocValueFieldsSearch", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getString("max_docvalue_fields_search"))
- .replace("#mappingTotalFieldsLimit", originalIndexSettings.getJSONObject(originalIndexForSettingsExtraction).getJSONObject("settings").getJSONObject("index").getJSONObject("mapping").getJSONObject("total_fields").getString("limit"));
- }
+ public static String buildIndexCreationRequest(String baseIndexSettings, String mapping, MigrationContext context, boolean isMonthlyIndex) throws IOException {
+ String settings = baseIndexSettings
+ .replace("#numberOfShards", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_SHARDS : NUMBER_OF_SHARDS))
+ .replace("#numberOfReplicas", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_REPLICAS : NUMBER_OF_REPLICAS))
+ .replace("#maxDocValueFieldsSearch", context.getConfigString(isMonthlyIndex ? MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH : MAX_DOC_VALUE_FIELDS_SEARCH))
+ .replace("#mappingTotalFieldsLimit", context.getConfigString(isMonthlyIndex ? MONTHLY_TOTAL_FIELDS_LIMIT : TOTAL_FIELDS_LIMIT));
return settings.replace("#mappings", mapping);
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index dcc2a0511..685d44288 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -1,4 +1,5 @@
import groovy.json.JsonSlurper
+import org.apache.unomi.shell.migration.service.MigrationConfig
import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.HttpRequestException
import org.apache.unomi.shell.migration.utils.HttpUtils
@@ -36,7 +37,7 @@ context.performMigrationStep("2.0.0-create-profileAlias-index", () -> {
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, aliasIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profileAlias.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, profileIndex, mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseRequest, mapping, context, false)
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + aliasIndex, newIndexSettings, null)
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
index 3f46f0dd6..d9aa5f889 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scopes.groovy
@@ -33,7 +33,7 @@ context.performMigrationStep("2.0.0-create-scope-index", () -> {
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseRequest, "${indexPrefix}-profile", mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseRequest, mapping, context, false)
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null)
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index 826243ae4..dd841ff11 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -26,6 +26,6 @@ String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBod
String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"];
indicesToReindex.each { indexToReindex ->
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "${indexToReindex}.json")
- String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-profile", mapping)
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReindex.toLowerCase()}", newIndexSettings, null, context)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
index 4711056c0..bcb4e6ae7 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-10-profileReindex.groovy
@@ -24,6 +24,6 @@ String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "profile.json")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, "${indexPrefix}-segment", mapping)
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, indexPrefix + "-profile",
newIndexSettings, MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/profile_migrate.painless"), context)
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index 941ff82a8..a9dc7ae1f 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -32,10 +32,8 @@ context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () ->
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String reIndexScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.0.0/event_migrate.painless");
String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json")
+String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, true)
Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-")
-// use session indices to extract monthly index settings
-Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-session-")
-String newIndexSettings = MigrationUtils.buildIndexCreationRequest(context.getHttpClient(), esAddress, baseSettings, sessionIndices[0], mapping)
eventIndices.each { eventIndex ->
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context)
}
\ No newline at end of file
diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index 5749e4cf9..f4cd52771 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -24,6 +24,14 @@ esLogin = ${org.apache.unomi.elasticsearch.username:-}
esPassword = ${org.apache.unomi.elasticsearch.password:-}
httpClient.trustAllCertificates = ${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false}
indexPrefix = ${org.apache.unomi.elasticsearch.index.prefix:-context}
+monthlyIndex.number_of_shards=${org.apache.unomi.elasticsearch.monthlyIndex.nbShards:-5}
+monthlyIndex.number_of_replicas=${org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas:-0}
+monthlyIndex.mapping.total_fields.limit=${org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000}
+monthlyIndex.max_docvalue_fields_search=${org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch:-1000}
+number_of_shards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5}
+number_of_replicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0}
+mapping.total_fields.limit=${org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit:-1000}
+max_docvalue_fields_search=${org.apache.unomi.elasticsearch.defaultIndex.indexMaxDocValueFieldsSearch:-1000}
# Should the migration try to recover from a previous run ?
# (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle)