You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2024/10/24 15:09:09 UTC
(unomi) branch master updated: UNOMI-861: use asynchronous delete_by_query in migration scripts to a… (#704)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 3b2da028d UNOMI-861: use asynchronous delete_by_query in migration scripts to a… (#704)
3b2da028d is described below
commit 3b2da028d6ac50b2b276c1150491bf0fc9cd689a
Author: jsinovassin <58...@users.noreply.github.com>
AuthorDate: Thu Oct 24 16:09:03 2024 +0100
UNOMI-861: use asynchronous delete_by_query in migration scripts to a… (#704)
* UNOMI-861: use asynchronous delete_by_query in migration scripts to avoid timeout
* UNOMI-861: use asynchronous update_by_query in migration scripts to avoid timeout
* UNOMI-861: surround code by try catch to avoid unwanted rerun
---
.../shell/migration/utils/MigrationUtils.java | 44 +++++++++++++++++++++-
.../internal/UnomiManagementServiceImpl.java | 28 +++++++++-----
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 4 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 4 +-
.../migrate-2.4.0-15-viewEventPagePath.groovy | 2 +-
.../migrate-2.5.0-10-loginEventScope.groovy | 2 +-
6 files changed, 67 insertions(+), 17 deletions(-)
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 2059f1dc8..69521c729 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -123,7 +123,11 @@ public class MigrationUtils {
});
String matchAllBodyRequest = resourceAsString(bundleContext, "requestBody/2.2.0/match_all_body_request.json");
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + lastIndexName + "/_delete_by_query", matchAllBodyRequest, null);
+ try {
+ deleteByQuery(httpClient, esAddress, lastIndexName, matchAllBodyRequest);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
@@ -290,6 +294,44 @@ public class MigrationUtils {
}
+ /**
+ * Updates documents in an index based on a specified query.
+ *
+ * <p>This method sends a request to update documents that match the provided query in the specified index. The update operation is
+ * performed asynchronously, and the method waits for the task to complete before returning.</p>
+ *
+ * @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server
+ * @param esAddress the address of the Elasticsearch server
+ * @param indexName the name of the index where documents should be updated
+ * @param requestBody the JSON body containing the query and update instructions for the documents
+ * @throws Exception if there is an error during the HTTP request or while waiting for the task to finish
+ */
+ public static void updateByQuery(CloseableHttpClient httpClient, String esAddress, String indexName, String requestBody) throws Exception {
+ JSONObject task = new JSONObject(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_update_by_query?wait_for_completion=false", requestBody, null));
+
+ //Wait for the deletion task to finish
+ waitForTaskToFinish(httpClient, esAddress, task.getString("task"), null);
+ }
+
+ /**
+ * Deletes documents from an index based on a specified query.
+ *
+ * <p>This method sends a request to the Elasticsearch cluster to delete documents
+ * that match the provided query in the specified index. The deletion operation is
+ * performed asynchronously, and the method waits for the task to complete before returning.</p>
+ *
+ * @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server
+ * @param esAddress the address of the Elasticsearch server
+ * @param indexName the name of the index from which documents should be deleted
+ * @param requestBody the JSON body containing the query that defines which documents to delete
+ * @throws Exception if there is an error during the HTTP request or while waiting for the task to finish
+ */
+ public static void deleteByQuery(CloseableHttpClient httpClient, String esAddress, String indexName, String requestBody) throws Exception {
+ JSONObject task = new JSONObject(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_delete_by_query?wait_for_completion=false", requestBody, null));
+ //Wait for the deletion task to finish
+ waitForTaskToFinish(httpClient, esAddress, task.getString("task"), null);
+ }
+
public static void waitForTaskToFinish(CloseableHttpClient httpClient, String esAddress, String taskId, MigrationContext migrationContext) throws IOException {
while (true) {
final JSONObject status = new JSONObject(
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
index defe72091..fbab16cd3 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
@@ -27,6 +27,8 @@ import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
@@ -38,26 +40,32 @@ import java.util.List;
@Component(service = UnomiManagementService.class, immediate = true)
public class UnomiManagementServiceImpl implements UnomiManagementService {
+ private static final Logger LOGGER = LoggerFactory.getLogger(UnomiManagementServiceImpl.class.getName());
+
private BundleContext bundleContext;
-
+
@Reference(cardinality = ReferenceCardinality.MANDATORY)
private MigrationService migrationService;
-
+
private final List<String> bundleSymbolicNames = new ArrayList<>();
private List<String> reversedBundleSymbolicNames;
@Activate
public void init(ComponentContext componentContext) throws Exception {
- this.bundleContext = componentContext.getBundleContext();
- initReversedBundleSymbolicNames();
+ try {
+ this.bundleContext = componentContext.getBundleContext();
+ initReversedBundleSymbolicNames();
- if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoMigrate"))) {
- migrationService.migrateUnomi(bundleContext.getProperty("unomi.autoMigrate"), true, null);
- }
+ if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoMigrate"))) {
+ migrationService.migrateUnomi(bundleContext.getProperty("unomi.autoMigrate"), true, null);
+ }
- if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) &&
- bundleContext.getProperty("unomi.autoStart").equals("true")) {
- startUnomi();
+ if (StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) &&
+ bundleContext.getProperty("unomi.autoStart").equals("true")) {
+ startUnomi();
+ }
+ } catch (Exception e) {
+ LOGGER.error("Error during Unomi startup when processing 'unomi.autoMigrate' or 'unomi.autoStart' properties:", e);
}
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index 685d44288..ad6863682 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -91,5 +91,5 @@ context.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles"
context.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
- HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
-})
\ No newline at end of file
+ MigrationUtils.deleteByQuery(context.getHttpClient(), esAddress, profileIndex, profileMergedDeleteRequest)
+})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index cb9f5ba1a..85a25df87 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -25,7 +25,7 @@ String indexPrefix = context.getConfigString("indexPrefix")
context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> {
String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json")
- HttpUtils.executePostRequest(context.getHttpClient(), "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null)
+ MigrationUtils.deleteByQuery(context.getHttpClient(), esAddress, "${indexPrefix}-event-*", removeInternalEventsRequest)
})
// Reindex the rest of the events
@@ -36,4 +36,4 @@ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings,
Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-date-")
eventIndices.each { eventIndex ->
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context)
-}
\ No newline at end of file
+}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
index 8eb0de38d..1ac1776b5 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
@@ -28,6 +28,6 @@ context.performMigrationStep("2.4.0-migrate-view-event-page-path", () -> {
String updatePathScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.4.0/view_event_page_path_migrate.painless")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.4.0/base_update_by_query_request.json")
eventIndices.each { eventIndex ->
- HttpUtils.executePostRequest(context.getHttpClient(), "${esAddress}/${eventIndex}/_update_by_query", baseSettings.replace('#painless', updatePathScript), null)
+ MigrationUtils.updateByQuery(context.getHttpClient(), esAddress, eventIndex, baseSettings.replace('#painless', updatePathScript));
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
index 24e2042e6..967bc5714 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
@@ -28,6 +28,6 @@ context.performMigrationStep("2.5.0-migrate-login-event-scope", () -> {
String updatePathScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.5.0/login_event_scope_migrate.painless")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.5.0/scope_update_by_query_request.json")
eventIndices.each { eventIndex ->
- HttpUtils.executePostRequest(context.getHttpClient(), "${esAddress}/${eventIndex}/_update_by_query", baseSettings.replace('#painless', updatePathScript), null)
+ MigrationUtils.updateByQuery(context.getHttpClient(), esAddress, eventIndex, baseSettings.replace('#painless', updatePathScript));
}
})