You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2023/02/27 17:10:47 UTC

[unomi] branch reduceIndicesItems3 updated: UNOMI-736: provide base migration for indices reduction

This is an automated email from the ASF dual-hosted git repository.

jkevan pushed a commit to branch reduceIndicesItems3
in repository https://gitbox.apache.org/repos/asf/unomi.git


The following commit(s) were added to refs/heads/reduceIndicesItems3 by this push:
     new 8361a522a UNOMI-736: provide base migration for indices reduction
8361a522a is described below

commit 8361a522a86b8fa296b979dba1944ac671cd7316
Author: Kevan <ke...@jahia.com>
AuthorDate: Mon Feb 27 18:10:37 2023 +0100

    UNOMI-736: provide base migration for indices reduction
---
 .../shell/migration/utils/MigrationUtils.java      | 13 ++--
 ...-2.2.0-00-rolloverAndMigrateEventSession.groovy |  4 +-
 .../migrate-2.2.0-05-indicesReduction.groovy       | 78 ++++++++++++++++++++++
 .../requestBody/2.2.0/base_reindex_request.json    |  2 +-
 .../requestBody/2.2.0/suffix_ids.painless          | 19 ++++++
 5 files changed, 108 insertions(+), 8 deletions(-)

diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index a12fd6ec7..2050d6ba5 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -173,8 +173,8 @@ public class MigrationUtils {
         return baseRequest.replace("#rolloverHotActions", rolloverHotActions.toString());
     }
 
-    public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName) throws Exception {
-        String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName);
+    public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName, String painlessScript) throws Exception {
+        String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName).replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : "");
 
         HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
     }
@@ -225,10 +225,13 @@ public class MigrationUtils {
                 HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
             }
         });
-        // Do a refresh
-        HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null);
 
-        waitForYellowStatus(httpClient, esAddress, migrationContext);
+        migrationContext.performMigrationStep("Reindex step for: " + indexName + " (refresh at the end)", () -> {
+            // Do a refresh
+            HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null);
+
+            waitForYellowStatus(httpClient, esAddress, migrationContext);
+        });
     }
 
     public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
index a04a4b097..c692fa7a2 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
@@ -58,7 +58,7 @@ Collections.sort(eventSortedIndices)
 context.performMigrationStep("2.2.0-migrate-existing-events", () -> {
     MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "event")
     eventSortedIndices.each { eventIndex ->
-        MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event")
+        MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event", null)
         sleep(3000)
     }
 })
@@ -86,7 +86,7 @@ Collections.sort(sessionSortedIndices)
 context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> {
     MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "session")
     sessionSortedIndices.each { sessionIndex ->
-        MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session")
+        MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session", null)
         sleep(3000)
     }
 })
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
new file mode 100644
index 000000000..8a07a5278
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
@@ -0,0 +1,78 @@
+import org.apache.unomi.shell.migration.service.MigrationContext
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
+String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
+
+context.performMigrationStep("2.2.0-create-systemItems-index", () -> {
+    if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, "${indexPrefix}-systemitems")) {
+        String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "systemItems.json")
+        String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
+        HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/${indexPrefix}-systemitems", newIndexSettings, null)
+    }
+})
+
+def indicesToReduce = [
+        actiontype: "systemitems",
+        campaign: "systemitems",
+        campaignevent: "systemitems",
+        goal: "systemitems",
+        userlist: "systemitems",
+        propertytype: "systemitems",
+        scope: "systemitems",
+        conditiontype: "systemitems",
+        rule: "systemitems",
+        scoring: "systemitems",
+        segment: "systemitems",
+        topic: "systemitems",
+        patch: "systemitems",
+        jsonschema: "systemitems",
+        importconfig: "systemitems",
+        exportconfig: "systemitems",
+        rulestats: "systemitems",
+        groovyaction: "systemitems",
+        persona: "profile",
+        personasession: "session"
+]
+def indicesToSuffixIds = [
+        rulestats: "-stat",
+        groovyaction: "-groovySourceCode"
+]
+indicesToReduce.each { indexToReduce ->
+    context.performMigrationStep("2.2.0-reduce-${indexToReduce.key}", () -> {
+        if (MigrationUtils.indexExists(context.getHttpClient(), esAddress, "${indexPrefix}-${indexToReduce.key}")) {
+            def painless = null
+            // check if we need to update the ids of those items first
+            if (indicesToSuffixIds.containsKey(indexToReduce.key)) {
+                painless = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.2.0/suffix_ids.painless").replace("#ID_SUFFIX", indicesToSuffixIds.get(indexToReduce.key))
+            }
+            // move items
+            MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReduce.key}", "${indexPrefix}-${indexToReduce.value}", painless)
+            MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, "${indexPrefix}-${indexToReduce.key}")
+            HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/${indexPrefix}-${indexToReduce.value}/_refresh", null, null);
+            MigrationUtils.waitForYellowStatus(context.getHttpClient(), esAddress, context);
+        }
+    })
+}
+
+
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
index ddcb79a5e..589e71084 100644
--- a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
@@ -4,5 +4,5 @@
   },
   "dest": {
     "index": "#dest"
-  }
+  }#painless
 }
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
new file mode 100644
index 000000000..3b63549f6
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ctx._id = ctx._id + '#ID_SUFFIX';
+ctx._source.itemId = ctx._id;
\ No newline at end of file