You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by jk...@apache.org on 2023/03/01 16:46:00 UTC
[unomi] branch master updated: UNOMI-736: merge all item indices into same index (#581)
This is an automated email from the ASF dual-hosted git repository.
jkevan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 9d1fc3222 UNOMI-736: merge all item indices into same index (#581)
9d1fc3222 is described below
commit 9d1fc322279dc424c07a7c2a4dbd95e8098e94de
Author: kevan Jahanshahi <jk...@apache.org>
AuthorDate: Wed Mar 1 17:45:53 2023 +0100
UNOMI-736: merge all item indices into same index (#581)
* DMF-5757: merge all item indices into same index for reducing number of indices
* fix bad merge
* small fix
* UNOMI-736: POC cleanup
* UNOMI-736: Fix old migration on 2.0.0 due to mappings removed from core persistence
* UNOMI-736: More fixes regarding groovy actions
* UNOMI-736: More fixes regarding groovy actions
* UNOMI-736: provide base migration for indices reduction
* UNOMI-736: provide base migration for indices reduction
* UNOMI-736: More fixes regarding groovy actions
* UNOMI-736: More fixes regarding groovy actions
* UNOMI-736: Fix IT
* UNOMI-736: Fix IT
* UNOMI-736: Fix IT
* UNOMI-736: Add logs to debug Github runs ... Something looks wrong
* UNOMI-736: Cleanup annoying blueprint stuff
* UNOMI-736: feedback
---
.../karaf-kar/src/main/feature/feature.xml | 8 +-
extensions/groovy-actions/services/pom.xml | 1 +
.../groovy/actions/GroovyActionDispatcher.java | 6 +-
.../actions/listener/GroovyActionListener.java | 30 ++---
.../services/impl/GroovyActionsServiceImpl.java | 98 +++++++-------
.../META-INF/cxs/mappings/groovyAction.json | 28 ----
.../resources/OSGI-INF/blueprint/blueprint.xml | 69 ----------
.../unomi/schema/impl/SchemaServiceImpl.java | 9 --
itests/pom.xml | 6 +
.../unomi/itests/GroovyActionsServiceIT.java | 13 +-
.../unomi/itests/migration/Migrate16xTo220IT.java | 23 ++--
kar/pom.xml | 10 --
.../ElasticSearchPersistenceServiceImpl.java | 78 ++++++++++--
.../META-INF/cxs/mappings/actionType.json | 20 ---
.../META-INF/cxs/mappings/campaignevent.json | 44 -------
.../META-INF/cxs/mappings/exportConfig.json | 20 ---
.../META-INF/cxs/mappings/importConfig.json | 20 ---
.../META-INF/cxs/mappings/jsonschema.json | 25 ----
.../resources/META-INF/cxs/mappings/persona.json | 38 ------
.../META-INF/cxs/mappings/personaSession.json | 41 ------
.../META-INF/cxs/mappings/propertyType.json | 61 ---------
.../resources/META-INF/cxs/mappings/rulestats.json | 20 ---
.../META-INF/cxs/mappings/systemItems.json | 141 +++++++++++++++++++++
.../resources/META-INF/cxs/mappings/topic.json | 20 ---
.../services/impl/rules/RulesServiceImpl.java | 24 +++-
.../shell/migration/utils/MigrationUtils.java | 13 +-
.../migrate-2.0.0-05-globalReindex.groovy | 2 +-
.../cxs/migration/migrate-2.0.0-20-scopes.groovy | 2 +-
...-2.2.0-00-rolloverAndMigrateEventSession.groovy | 4 +-
.../migrate-2.2.0-05-indicesReduction.groovy | 78 ++++++++++++
.../requestBody/2.0.0}/mappings/campaign.json | 0
.../requestBody/2.0.0}/mappings/conditionType.json | 0
.../requestBody/2.0.0}/mappings/goal.json | 0
.../requestBody/2.0.0}/mappings/patch.json | 0
.../requestBody/2.0.0}/mappings/rule.json | 0
.../requestBody/2.0.0}/mappings/scope.json | 0
.../requestBody/2.0.0}/mappings/scoring.json | 0
.../requestBody/2.0.0}/mappings/segment.json | 0
.../requestBody/2.2.0/base_reindex_request.json | 2 +-
.../requestBody/2.2.0/suffix_ids.painless | 19 +++
40 files changed, 434 insertions(+), 539 deletions(-)
diff --git a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
index a4af565e2..649b74097 100644
--- a/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
+++ b/extensions/groovy-actions/karaf-kar/src/main/feature/feature.xml
@@ -18,15 +18,15 @@
<features xmlns="http://karaf.apache.org/xmlns/features/v1.6.0" name="unomi-groovy-actions">
<feature name="unomi-groovy-actions" description="${project.name}" version="${project.version}">
<details>${project.description}</details>
- <feature prerequisite="true" dependency="false">wrap</feature>
- <feature dependency="true">unomi-kar</feature>
+ <feature>wrap</feature>
+ <feature>unomi-kar</feature>
<bundle start-level="85">mvn:org.codehaus.groovy/groovy/${groovy.version}</bundle>
<bundle start-level="85">mvn:org.codehaus.groovy/groovy-xml/${groovy.version}</bundle>
<bundle start-level="85">mvn:org.codehaus.groovy/groovy-json/${groovy.version}</bundle>
- <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-services/${project.version}</bundle>
- <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-rest/${project.version}</bundle>
<bundle start-level="85">wrap:mvn:io.github.http-builder-ng/http-builder-ng-core/1.0.4</bundle>
<bundle start-level="85">mvn:org.jsoup/jsoup/1.13.1</bundle>
<bundle start-level="85">mvn:com.sun.activation/javax.activation/1.2.0</bundle>
+ <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-services/${project.version}</bundle>
+ <bundle start-level="85" start="false">mvn:org.apache.unomi/unomi-groovy-actions-rest/${project.version}</bundle>
</feature>
</features>
diff --git a/extensions/groovy-actions/services/pom.xml b/extensions/groovy-actions/services/pom.xml
index 8494a9938..b9b726575 100644
--- a/extensions/groovy-actions/services/pom.xml
+++ b/extensions/groovy-actions/services/pom.xml
@@ -104,6 +104,7 @@
<extensions>true</extensions>
<configuration>
<instructions>
+ <_dsannotations>*</_dsannotations>
<Embed-Dependency>*;scope=compile|runtime</Embed-Dependency>
<Unomi-Source-Folders>${project.basedir}</Unomi-Source-Folders>
<DynamicImport-Package>*</DynamicImport-Package>
diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java
index dadeabb44..f5276ab37 100644
--- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java
+++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/GroovyActionDispatcher.java
@@ -25,6 +25,8 @@ import org.apache.unomi.api.actions.ActionDispatcher;
import org.apache.unomi.groovy.actions.services.GroovyActionsService;
import org.apache.unomi.metrics.MetricAdapter;
import org.apache.unomi.metrics.MetricsService;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,6 +34,7 @@ import org.slf4j.LoggerFactory;
* An implementation of an ActionDispatcher for the Groovy language. This dispatcher will load the groovy action script matching to an
* actionName. If a script if found, it will be executed.
*/
+@Component(service = ActionDispatcher.class)
public class GroovyActionDispatcher implements ActionDispatcher {
private static final Logger logger = LoggerFactory.getLogger(GroovyActionDispatcher.class.getName());
@@ -39,13 +42,14 @@ public class GroovyActionDispatcher implements ActionDispatcher {
private static final String GROOVY_PREFIX = "groovy";
private MetricsService metricsService;
-
private GroovyActionsService groovyActionsService;
+ @Reference
public void setMetricsService(MetricsService metricsService) {
this.metricsService = metricsService;
}
+ @Reference
public void setGroovyActionsService(GroovyActionsService groovyActionsService) {
this.groovyActionsService = groovyActionsService;
}
diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java
index 2ac3e37c4..1dacf8af4 100644
--- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java
+++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/listener/GroovyActionListener.java
@@ -16,16 +16,17 @@
*/
package org.apache.unomi.groovy.actions.listener;
-import groovy.util.GroovyScriptEngine;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
-import org.apache.unomi.groovy.actions.GroovyAction;
import org.apache.unomi.groovy.actions.services.GroovyActionsService;
-import org.apache.unomi.persistence.spi.PersistenceService;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
import org.osgi.framework.BundleEvent;
import org.osgi.framework.SynchronousBundleListener;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Deactivate;
+import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,30 +40,24 @@ import java.util.Enumeration;
* The description of the action will be loaded from the ActionDescriptor annotation present in the groovy file.
* The script will be stored in the ES index groovyAction
*/
+@Component(service = SynchronousBundleListener.class)
public class GroovyActionListener implements SynchronousBundleListener {
private static final Logger logger = LoggerFactory.getLogger(GroovyActionListener.class.getName());
public static final String ENTRIES_LOCATION = "META-INF/cxs/actions";
- private PersistenceService persistenceService;
private GroovyActionsService groovyActionsService;
private BundleContext bundleContext;
- public void setPersistenceService(PersistenceService persistenceService) {
- this.persistenceService = persistenceService;
- }
-
+ @Reference
public void setGroovyActionsService(GroovyActionsService groovyActionsService) {
this.groovyActionsService = groovyActionsService;
}
- public void setBundleContext(BundleContext bundleContext) {
+ @Activate
+ public void postConstruct(BundleContext bundleContext) {
this.bundleContext = bundleContext;
- }
-
- public void postConstruct() {
logger.debug("postConstruct {}", bundleContext.getBundle());
- createIndex();
loadGroovyActions(bundleContext);
for (Bundle bundle : bundleContext.getBundles()) {
if (bundle.getBundleContext() != null && bundle.getBundleId() != bundleContext.getBundle().getBundleId()) {
@@ -74,6 +69,7 @@ public class GroovyActionListener implements SynchronousBundleListener {
logger.info("Groovy Action Dispatcher initialized.");
}
+ @Deactivate
public void preDestroy() {
processBundleStop(bundleContext);
bundleContext.removeBundleListener(this);
@@ -107,14 +103,6 @@ public class GroovyActionListener implements SynchronousBundleListener {
}
}
- public void createIndex() {
- if (persistenceService.createIndex(GroovyAction.ITEM_TYPE)) {
- logger.info("GroovyAction index created");
- } else {
- logger.info("GroovyAction index already exists");
- }
- }
-
private void addGroovyAction(URL groovyActionURL) {
try {
groovyActionsService.save(FilenameUtils.getName(groovyActionURL.getPath()).replace(".groovy", ""),
diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java
index b72a089b4..5761cc1c5 100644
--- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java
+++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/impl/GroovyActionsServiceImpl.java
@@ -35,7 +35,9 @@ import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.control.customizers.ImportCustomizer;
import org.osgi.framework.BundleContext;
import org.osgi.framework.wiring.BundleWiring;
-import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.component.annotations.*;
+import org.osgi.service.metatype.annotations.Designate;
+import org.osgi.service.metatype.annotations.ObjectClassDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,8 +57,15 @@ import static java.util.Arrays.asList;
/**
* Implementation of the GroovyActionService. Allows to create a groovy action from a groovy file
*/
+@Component(service = GroovyActionsService.class, configurationPid = "org.apache.unomi.groovy.actions")
+@Designate(ocd = GroovyActionsServiceImpl.GroovyActionsServiceConfig.class)
public class GroovyActionsServiceImpl implements GroovyActionsService {
+ @ObjectClassDefinition(name = "Groovy actions service config", description = "The configuration for the Groovy actions service")
+ public @interface GroovyActionsServiceConfig {
+ int services_groovy_actions_refresh_interval() default 1000;
+ }
+
private BundleContext bundleContext;
private GroovyScriptEngine groovyScriptEngine;
@@ -70,41 +79,30 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
private static final Logger logger = LoggerFactory.getLogger(GroovyActionsServiceImpl.class.getName());
private static final String BASE_SCRIPT_NAME = "BaseScript";
+ private static final String GROOVY_SOURCE_CODE_ID_SUFFIX = "-groovySourceCode";
- public void setBundleContext(BundleContext bundleContext) {
- this.bundleContext = bundleContext;
- }
-
- @Reference
private DefinitionsService definitionsService;
-
- @Reference
private PersistenceService persistenceService;
-
- @Reference
private SchedulerService schedulerService;
-
- @Reference
private ActionExecutorDispatcher actionExecutorDispatcher;
+ private GroovyActionsServiceConfig config;
- private Integer groovyActionsRefreshInterval = 1000;
-
+ @Reference
public void setDefinitionsService(DefinitionsService definitionsService) {
this.definitionsService = definitionsService;
}
+ @Reference
public void setPersistenceService(PersistenceService persistenceService) {
this.persistenceService = persistenceService;
}
- public void setGroovyActionsRefreshInterval(Integer groovyActionsRefreshInterval) {
- this.groovyActionsRefreshInterval = groovyActionsRefreshInterval;
- }
-
+ @Reference
public void setSchedulerService(SchedulerService schedulerService) {
this.schedulerService = schedulerService;
}
+ @Reference
public void setActionExecutorDispatcher(ActionExecutorDispatcher actionExecutorDispatcher) {
this.actionExecutorDispatcher = actionExecutorDispatcher;
}
@@ -113,13 +111,17 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
return groovyShell;
}
- public void postConstruct() {
+ @Activate
+ public void start(GroovyActionsServiceConfig config, BundleContext bundleContext) {
logger.debug("postConstruct {}", bundleContext.getBundle());
- groovyCodeSourceMap = new HashMap<>();
- GroovyBundleResourceConnector bundleResourceConnector = new GroovyBundleResourceConnector(bundleContext);
+ this.config = config;
+ this.bundleContext = bundleContext;
+ this.groovyCodeSourceMap = new HashMap<>();
+
+ GroovyBundleResourceConnector bundleResourceConnector = new GroovyBundleResourceConnector(bundleContext);
GroovyClassLoader groovyLoader = new GroovyClassLoader(bundleContext.getBundle().adapt(BundleWiring.class).getClassLoader());
- groovyScriptEngine = new GroovyScriptEngine(bundleResourceConnector, groovyLoader);
+ this.groovyScriptEngine = new GroovyScriptEngine(bundleResourceConnector, groovyLoader);
initializeGroovyShell();
try {
@@ -131,9 +133,12 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
logger.info("Groovy action service initialized.");
}
+ @Deactivate
public void onDestroy() {
logger.debug("onDestroy Method called");
- scheduledFuture.cancel(true);
+ if (scheduledFuture != null && !scheduledFuture.isCancelled()) {
+ scheduledFuture.cancel(true);
+ }
}
/**
@@ -153,9 +158,7 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
return;
}
logger.debug("Found Groovy base script at {}, loading... ", groovyBaseScriptURL.getPath());
- GroovyCodeSource groovyCodeSource = new GroovyCodeSource(IOUtils.toString(groovyBaseScriptURL.openStream()), BASE_SCRIPT_NAME,
- "/groovy/script");
- groovyCodeSourceMap.put(BASE_SCRIPT_NAME, groovyCodeSource);
+ GroovyCodeSource groovyCodeSource = new GroovyCodeSource(IOUtils.toString(groovyBaseScriptURL.openStream()), BASE_SCRIPT_NAME, "/groovy/script");
groovyScriptEngine.getGroovyClassLoader().parseClass(groovyCodeSource, true);
}
@@ -183,15 +186,10 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
@Override
public void save(String actionName, String groovyScript) {
- handleFile(actionName, groovyScript);
- }
-
- private void handleFile(String actionName, String groovyScript) {
GroovyCodeSource groovyCodeSource = buildClassScript(groovyScript, actionName);
try {
saveActionType(groovyShell.parse(groovyCodeSource).getClass().getMethod("execute").getAnnotation(Action.class));
saveScript(actionName, groovyScript);
- groovyCodeSourceMap.put(actionName, groovyCodeSource);
logger.info("The script {} has been loaded.", actionName);
} catch (NoSuchMethodException e) {
logger.error("Failed to save the script {}", actionName, e);
@@ -219,19 +217,21 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
@Override
public void remove(String id) {
- try {
- definitionsService.removeActionType(
- groovyShell.parse(groovyCodeSourceMap.get(id)).getClass().getMethod("execute").getAnnotation(Action.class).id());
- } catch (NoSuchMethodException e) {
- logger.error("Failed to delete the action type for the id {}", id, e);
+ String groovySourceCodeId = getGroovyCodeSourceIdForActionId(id);
+ if (groovyCodeSourceMap.containsKey(groovySourceCodeId)) {
+ try {
+ definitionsService.removeActionType(
+ groovyShell.parse(groovyCodeSourceMap.get(groovySourceCodeId)).getClass().getMethod("execute").getAnnotation(Action.class).id());
+ } catch (NoSuchMethodException e) {
+ logger.error("Failed to delete the action type for the id {}", id, e);
+ }
+ persistenceService.remove(groovySourceCodeId, GroovyAction.class);
}
- persistenceService.remove(id, GroovyAction.class);
- groovyCodeSourceMap.remove(id);
}
@Override
public GroovyCodeSource getGroovyCodeSource(String id) {
- return groovyCodeSourceMap.get(id);
+ return groovyCodeSourceMap.get(getGroovyCodeSourceIdForActionId(id));
}
/**
@@ -245,15 +245,25 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
return new GroovyCodeSource(groovyScript, actionName, "/groovy/script");
}
- private void saveScript(String name, String script) {
- GroovyAction groovyScript = new GroovyAction(name, script);
+ /**
+ * We use a suffix for avoiding id conflict between the actionType and the groovyAction in ElasticSearch
+ * Since those items are now stored in the same ES index
+ * @param actionName name/id of the actionType
+ * @return id of the groovyAction source code for query/save/storage usage.
+ */
+ private String getGroovyCodeSourceIdForActionId(String actionName) {
+ return actionName + GROOVY_SOURCE_CODE_ID_SUFFIX;
+ }
+
+ private void saveScript(String actionName, String script) {
+ String groovyName = getGroovyCodeSourceIdForActionId(actionName);
+ GroovyAction groovyScript = new GroovyAction(groovyName, script);
persistenceService.save(groovyScript);
+ logger.info("The script {} has been persisted.", groovyName);
}
private void refreshGroovyActions() {
Map<String, GroovyCodeSource> refreshedGroovyCodeSourceMap = new HashMap<>();
- GroovyCodeSource baseScript = groovyCodeSourceMap.get(BASE_SCRIPT_NAME);
- refreshedGroovyCodeSourceMap.put(BASE_SCRIPT_NAME, baseScript);
persistenceService.getAllItems(GroovyAction.class).forEach(groovyAction -> refreshedGroovyCodeSourceMap
.put(groovyAction.getName(), buildClassScript(groovyAction.getScript(), groovyAction.getName())));
groovyCodeSourceMap = refreshedGroovyCodeSourceMap;
@@ -266,7 +276,7 @@ public class GroovyActionsServiceImpl implements GroovyActionsService {
refreshGroovyActions();
}
};
- scheduledFuture = schedulerService.getScheduleExecutorService().scheduleWithFixedDelay(task, 0, groovyActionsRefreshInterval,
+ scheduledFuture = schedulerService.getScheduleExecutorService().scheduleWithFixedDelay(task, 0, config.services_groovy_actions_refresh_interval(),
TimeUnit.MILLISECONDS);
}
}
diff --git a/extensions/groovy-actions/services/src/main/resources/META-INF/cxs/mappings/groovyAction.json b/extensions/groovy-actions/services/src/main/resources/META-INF/cxs/mappings/groovyAction.json
deleted file mode 100644
index 6cd8d3ff8..000000000
--- a/extensions/groovy-actions/services/src/main/resources/META-INF/cxs/mappings/groovyAction.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ],
- "properties": {
- "name": {
- "type": "text"
- },
- "script": {
- "type": "text"
- }
- }
-}
\ No newline at end of file
diff --git a/extensions/groovy-actions/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/extensions/groovy-actions/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
deleted file mode 100644
index 433cd549c..000000000
--- a/extensions/groovy-actions/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<blueprint xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns:cm="http://aries.apache.org/blueprint/xmlns/blueprint-cm/v1.1.0"
- xmlns="http://www.osgi.org/xmlns/blueprint/v1.0.0"
- xsi:schemaLocation="http://www.osgi.org/xmlns/blueprint/v1.0.0 http://www.osgi.org/xmlns/blueprint/v1.0.0/blueprint.xsd">
-
- <cm:property-placeholder persistent-id="org.apache.unomi.groovy.actions" update-strategy="reload">
- <cm:default-properties>
- <cm:property name="services.groovy.actions.refresh.interval" value="1000"/>
- </cm:default-properties>
- </cm:property-placeholder>
-
- <reference id="metricsService" interface="org.apache.unomi.metrics.MetricsService"/>
- <reference id="definitionsService" interface="org.apache.unomi.api.services.DefinitionsService"/>
- <reference id="persistenceService" interface="org.apache.unomi.persistence.spi.PersistenceService"/>
- <reference id="schedulerService" interface="org.apache.unomi.api.services.SchedulerService"/>
- <reference id="actionExecutorDispatcher" interface="org.apache.unomi.services.actions.ActionExecutorDispatcher"/>
-
- <bean id="groovyActionsServiceImpl" class="org.apache.unomi.groovy.actions.services.impl.GroovyActionsServiceImpl"
- init-method="postConstruct" destroy-method="onDestroy">
- <property name="bundleContext" ref="blueprintBundleContext"/>
- <property name="definitionsService" ref="definitionsService"/>
- <property name="persistenceService" ref="persistenceService"/>
- <property name="schedulerService" ref="schedulerService"/>
- <property name="actionExecutorDispatcher" ref="actionExecutorDispatcher"/>
- <property name="groovyActionsRefreshInterval" value="${services.groovy.actions.refresh.interval}"/>
- </bean>
- <service id="groovyActionsService" ref="groovyActionsServiceImpl"
- interface="org.apache.unomi.groovy.actions.services.GroovyActionsService"/>
-
- <bean id="groovyActionDispatcherImpl" class="org.apache.unomi.groovy.actions.GroovyActionDispatcher">
- <property name="metricsService" ref="metricsService"/>
- <property name="groovyActionsService" ref="groovyActionsServiceImpl"/>
- </bean>
- <service id="groovyActionDispatcher" ref="groovyActionDispatcherImpl">
- <interfaces>
- <value>org.apache.unomi.api.actions.ActionDispatcher</value>
- </interfaces>
- </service>
-
- <bean id="groovyActionListenerImpl" class="org.apache.unomi.groovy.actions.listener.GroovyActionListener"
- init-method="postConstruct" destroy-method="preDestroy">
- <property name="persistenceService" ref="persistenceService"/>
- <property name="bundleContext" ref="blueprintBundleContext"/>
- <property name="groovyActionsService" ref="groovyActionsServiceImpl"/>
- </bean>
- <service id="groovyActionListener" ref="groovyActionListenerImpl">
- <interfaces>
- <value>org.osgi.framework.SynchronousBundleListener</value>
- </interfaces>
- </service>
-</blueprint>
diff --git a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
index ebc431f7f..44ff90f7b 100644
--- a/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
+++ b/extensions/json-schema/services/src/main/java/org/apache/unomi/schema/impl/SchemaServiceImpl.java
@@ -296,14 +296,6 @@ public class SchemaServiceImpl implements SchemaService {
return schema;
}
- private void initPersistenceIndex() {
- if (persistenceService.createIndex(JsonSchemaWrapper.ITEM_TYPE)) {
- logger.info("{} index created", JsonSchemaWrapper.ITEM_TYPE);
- } else {
- logger.info("{} index already exists", JsonSchemaWrapper.ITEM_TYPE);
- }
- }
-
private void initTimers() {
TimerTask task = new TimerTask() {
@Override
@@ -345,7 +337,6 @@ public class SchemaServiceImpl implements SchemaService {
public void init() {
scheduler = Executors.newSingleThreadScheduledExecutor();
- initPersistenceIndex();
initJsonSchemaFactory();
initTimers();
logger.info("Schema service initialized.");
diff --git a/itests/pom.xml b/itests/pom.xml
index f0a602efe..375e816b6 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -133,6 +133,12 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.unomi</groupId>
+ <artifactId>unomi-groovy-actions-services</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
diff --git a/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java
index 20c507f8d..7fecea183 100644
--- a/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/GroovyActionsServiceIT.java
@@ -101,10 +101,10 @@ public class GroovyActionsServiceIT extends BaseIT {
groovyActionsService.save(UPDATE_ADDRESS_ACTION, loadGroovyAction(UPDATE_ADDRESS_ACTION_GROOVY_FILE));
keepTrying("Failed waiting for the creation of the GroovyAction for the trigger action test",
- () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, 1000, 100);
+ () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
ActionType actionType = keepTrying("Failed waiting for the creation of the GroovyAction for trigger action test",
- () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, 1000, 100);
+ () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
Assert.assertNotNull(actionType);
@@ -118,9 +118,12 @@ public class GroovyActionsServiceIT extends BaseIT {
groovyActionsService.save(UPDATE_ADDRESS_ACTION, loadGroovyAction(UPDATE_ADDRESS_ACTION_GROOVY_FILE));
ActionType actionType = keepTrying("Failed waiting for the creation of the GroovyAction for the save test",
- () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, 1000, 100);
+ () -> definitionsService.getActionType(UPDATE_ADDRESS_GROOVY_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
- Assert.assertEquals(UPDATE_ADDRESS_ACTION, groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION).getName());
+ GroovyCodeSource groovyCodeSource = keepTrying("Failed waiting for the creation of the GroovyAction for the save test",
+ () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ Assert.assertEquals(UPDATE_ADDRESS_ACTION + "-groovySourceCode", groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION).getName());
Assert.assertTrue(actionType.getMetadata().getId().contains(UPDATE_ADDRESS_GROOVY_ACTION));
Assert.assertEquals(2, actionType.getMetadata().getSystemTags().size());
@@ -137,7 +140,7 @@ public class GroovyActionsServiceIT extends BaseIT {
groovyActionsService.save(UPDATE_ADDRESS_ACTION, loadGroovyAction(UPDATE_ADDRESS_ACTION_GROOVY_FILE));
GroovyCodeSource groovyCodeSource = keepTrying("Failed waiting for the creation of the GroovyAction for the remove test",
- () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, 1000, 100);
+ () -> groovyActionsService.getGroovyCodeSource(UPDATE_ADDRESS_ACTION), Objects::nonNull, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
Assert.assertNotNull(groovyCodeSource);
diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
index a3e320a3c..8ae93daf4 100644
--- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java
@@ -40,6 +40,7 @@ public class Migrate16xTo220IT extends BaseIT {
private int sessionCount = 0;
private static final int NUMBER_DUPLICATE_SESSIONS = 3;
+ private static final int NUMBER_PERSONA_SESSIONS = 2;
@Override
@Before
public void waitForStartup() throws InterruptedException {
@@ -96,6 +97,9 @@ public class Migrate16xTo220IT extends BaseIT {
/**
* Checks if at least the new index for events and sessions exists.
+ * Also checks:
+ * - duplicated sessions are correctly removed (-3 sessions in final count)
+ * - persona sessions are now merged in session index due to index reduction in 2_2_0 (+2 sessions in final count)
*/
private void checkEventSessionRollover2_2_0() throws IOException {
Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400", "context-event-000001"));
@@ -113,23 +117,20 @@ public class Migrate16xTo220IT extends BaseIT {
newSessioncount += jsonNode.get("count").asInt();
}
Assert.assertEquals(eventCount, newEventcount);
- Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS, newSessioncount);
+ Assert.assertEquals(sessionCount - NUMBER_DUPLICATE_SESSIONS + NUMBER_PERSONA_SESSIONS, newSessioncount);
}
/**
* Multiple index mappings have been update, check a simple check that after migration those mappings contains the latest modifications.
*/
private void checkForMappingUpdates() throws IOException {
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scope/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-profilealias/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-segment/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-scoring/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-campaign/_mapping", null).contains("\"entryCondition\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-conditiontype/_mapping", null).contains("\"parentCondition\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-goal/_mapping", null).contains("\"startEvent\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-patch/_mapping", null).contains("\"data\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-rule/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
- Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-rule/_mapping", null).contains("\"parameterValues\":{\"type\":\"object\",\"enabled\":false}"));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"match\":\"*\",\"match_mapping_type\":\"string\",\"mapping\":{\"analyzer\":\"folding\""));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"condition\":{\"type\":\"object\",\"enabled\":false}"));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"entryCondition\":{\"type\":\"object\",\"enabled\":false}"));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"parentCondition\":{\"type\":\"object\",\"enabled\":false}"));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"startEvent\":{\"type\":\"object\",\"enabled\":false}"));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"data\":{\"type\":\"object\",\"enabled\":false}"));
+ Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-systemitems/_mapping", null).contains("\"parameterValues\":{\"type\":\"object\",\"enabled\":false}"));
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/context-profile/_mapping", null).contains("\"interests\":{\"type\":\"nested\""));
for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400", "context-event-")) {
Assert.assertTrue(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/" + eventIndex + "/_mapping", null).contains("\"flattenedProperties\":{\"type\":\"flattened\"}"));
diff --git a/kar/pom.xml b/kar/pom.xml
index 6e41b8097..3e72e3994 100644
--- a/kar/pom.xml
+++ b/kar/pom.xml
@@ -124,16 +124,6 @@
<artifactId>unomi-json-schema-rest</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.unomi</groupId>
- <artifactId>unomi-groovy-actions-services</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.unomi</groupId>
- <artifactId>unomi-groovy-actions-rest</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.unomi</groupId>
<artifactId>cxs-privacy-extension-services</artifactId>
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index ecc703511..13a802db0 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -254,6 +254,34 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private Map<String, Map<String, Map<String, Object>>> knownMappings = new HashMap<>();
+ private static final Map<String, String> itemTypeIndexNameMap = new HashMap<>();
+ static {
+ itemTypeIndexNameMap.put("actionType", "systemItems");
+ itemTypeIndexNameMap.put("campaign", "systemItems");
+ itemTypeIndexNameMap.put("campaignevent", "systemItems");
+ itemTypeIndexNameMap.put("goal", "systemItems");
+ itemTypeIndexNameMap.put("userList", "systemItems");
+ itemTypeIndexNameMap.put("propertyType", "systemItems");
+ itemTypeIndexNameMap.put("scope", "systemItems");
+ itemTypeIndexNameMap.put("conditionType", "systemItems");
+ itemTypeIndexNameMap.put("rule", "systemItems");
+ itemTypeIndexNameMap.put("scoring", "systemItems");
+ itemTypeIndexNameMap.put("segment", "systemItems");
+ itemTypeIndexNameMap.put("groovyAction", "systemItems");
+ itemTypeIndexNameMap.put("topic", "systemItems");
+ itemTypeIndexNameMap.put("patch", "systemItems");
+ itemTypeIndexNameMap.put("jsonSchema", "systemItems");
+ itemTypeIndexNameMap.put("importConfig", "systemItems");
+ itemTypeIndexNameMap.put("exportConfig", "systemItems");
+ itemTypeIndexNameMap.put("rulestats", "systemItems");
+
+ itemTypeIndexNameMap.put("profile", "profile");
+ itemTypeIndexNameMap.put("persona", "profile");
+
+ itemTypeIndexNameMap.put("session", "session");
+ itemTypeIndexNameMap.put("personaSession", "session");
+ }
+
public void setBundleContext(BundleContext bundleContext) {
this.bundleContext = bundleContext;
}
@@ -1099,13 +1127,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
for (int i = 0; i < scripts.length; i++) {
RefreshRequest refreshRequest = new RefreshRequest(index);
client.indices().refresh(refreshRequest, RequestOptions.DEFAULT);
+ QueryBuilder queryBuilder = conditionESQueryBuilderDispatcher.buildFilter(conditions[i]);
UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(index);
updateByQueryRequest.setConflicts("proceed");
updateByQueryRequest.setMaxRetries(1000);
updateByQueryRequest.setSlices(2);
updateByQueryRequest.setScript(scripts[i]);
- updateByQueryRequest.setQuery(conditionESQueryBuilderDispatcher.buildFilter(conditions[i]));
+ updateByQueryRequest.setQuery(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, queryBuilder) : queryBuilder);
BulkByScrollResponse response = client.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT);
@@ -1264,8 +1293,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
protected Boolean execute(Object... args) throws Exception {
try {
String itemType = Item.getItemType(clazz);
+ QueryBuilder queryBuilder = conditionESQueryBuilderDispatcher.getQueryBuilder(query);
final DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(getIndexNameForQuery(itemType))
- .setQuery(conditionESQueryBuilderDispatcher.getQueryBuilder(query))
+ .setQuery(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, queryBuilder) : queryBuilder)
// Setting slices to auto will let Elasticsearch choose the number of slices to use.
// This setting will use one slice per shard, up to a certain limit.
// The delete request will be more efficient and faster than no slicing.
@@ -1903,7 +1933,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
CountRequest countRequest = new CountRequest(getIndexNameForQuery(itemType));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
- searchSourceBuilder.query(filter);
+ searchSourceBuilder.query(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, filter) : filter);
countRequest.source(searchSourceBuilder);
CountResponse response = client.count(countRequest, RequestOptions.DEFAULT);
return response.getCount();
@@ -1938,7 +1968,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.fetchSource(true)
.seqNoAndPrimaryTerm(true)
- .query(query)
+ .query(isItemTypeSharingIndex(itemType) ? wrapWithItemTypeQuery(itemType, query) : query)
.size(size < 0 ? defaultQueryLimit : size)
.from(offset);
if (scrollTimeValidity != null) {
@@ -2159,7 +2189,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
- searchSourceBuilder.query(QueryBuilders.matchAllQuery());
+ MatchAllQueryBuilder matchAll = QueryBuilders.matchAllQuery();
+ boolean isItemTypeSharingIndex = isItemTypeSharingIndex(itemType);
+ searchSourceBuilder.query(isItemTypeSharingIndex ? getItemTypeQueryBuilder(itemType) : matchAll);
List<AggregationBuilder> lastAggregation = new ArrayList<AggregationBuilder>();
if (aggregate != null) {
@@ -2240,11 +2272,15 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
if (filter != null) {
- searchSourceBuilder.query(conditionESQueryBuilderDispatcher.buildFilter(filter));
+ searchSourceBuilder.query(isItemTypeSharingIndex ?
+ wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter)) :
+ conditionESQueryBuilderDispatcher.buildFilter(filter));
}
} else {
if (filter != null) {
- AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", conditionESQueryBuilderDispatcher.buildFilter(filter));
+ AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", isItemTypeSharingIndex ?
+ wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter)) :
+ conditionESQueryBuilderDispatcher.buildFilter(filter));
for (AggregationBuilder aggregationBuilder : lastAggregation) {
filterAggregation.subAggregation(aggregationBuilder);
}
@@ -2465,7 +2501,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()
.size(0)
- .query(QueryBuilders.matchAllQuery());
+ .query(isItemTypeSharingIndex(itemType) ? getItemTypeQueryBuilder(itemType) : QueryBuilders.matchAllQuery());
+
AggregationBuilder filterAggregation = AggregationBuilders.filter("metrics", conditionESQueryBuilderDispatcher.buildFilter(condition));
if (metrics != null) {
@@ -2598,13 +2635,32 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
private String getIndexNameForQuery(String itemType) {
return isItemTypeRollingOver(itemType) ? getRolloverIndexForQuery(itemType) : getIndex(itemType);
}
-
+
private String getRolloverIndexForQuery(String itemType) {
return indexPrefix + "-" + itemType.toLowerCase() + "-*";
}
- private String getIndex(String indexItemTypePart) {
- return (indexPrefix + "-" + indexItemTypePart).toLowerCase();
+ private String getIndex(String itemType) {
+ return (indexPrefix + "-" + getIndexNameForItemType(itemType)).toLowerCase();
+ }
+
+ private String getIndexNameForItemType(String itemType) {
+ return itemTypeIndexNameMap.getOrDefault(itemType, itemType);
+ }
+
+ private QueryBuilder wrapWithItemTypeQuery(String itemType, QueryBuilder originalQuery) {
+ BoolQueryBuilder wrappedQuery = QueryBuilders.boolQuery();
+ wrappedQuery.must(getItemTypeQueryBuilder(itemType));
+ wrappedQuery.must(originalQuery);
+ return wrappedQuery;
+ }
+
+ private QueryBuilder getItemTypeQueryBuilder(String itemType) {
+ return QueryBuilders.termQuery("itemType", ConditionContextHelper.foldToASCII(itemType));
+ }
+
+ private boolean isItemTypeSharingIndex(String itemType) {
+ return itemTypeIndexNameMap.containsKey(itemType);
}
private boolean isItemTypeRollingOver(String itemType) {
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
deleted file mode 100644
index e1ac5f8d4..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/actionType.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
deleted file mode 100644
index 7d6a564d0..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaignevent.json
+++ /dev/null
@@ -1,44 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ],
- "properties": {
- "cost": {
- "type": "double"
- },
- "eventDate": {
- "type": "date"
- },
- "metadata": {
- "properties": {
- "enabled": {
- "type": "boolean"
- },
- "hidden": {
- "type": "boolean"
- },
- "missingPlugins": {
- "type": "boolean"
- },
- "readOnly": {
- "type": "boolean"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/exportConfig.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/exportConfig.json
deleted file mode 100644
index e1ac5f8d4..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/exportConfig.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/importConfig.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/importConfig.json
deleted file mode 100644
index e1ac5f8d4..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/importConfig.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/jsonschema.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/jsonschema.json
deleted file mode 100644
index 896e1be21..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/jsonschema.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ],
- "properties": {
- "schema": {
- "type": "text"
- }
- }
-}
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
deleted file mode 100644
index d0142451f..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/persona.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ],
- "properties": {
- "properties": {
- "properties": {
- "firstVisit": {
- "type": "date"
- },
- "lastVisit": {
- "type": "date"
- },
- "previousVisit": {
- "type": "date"
- },
- "nbOfVisits": {
- "type": "long"
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
deleted file mode 100644
index c635e0285..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/personaSession.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ],
- "properties": {
- "duration": {
- "type": "long"
- },
- "timeStamp": {
- "type": "date"
- },
- "lastEventDate": {
- "type": "date"
- },
- "properties": {
- "properties": {
- "location": {
- "type": "geo_point"
- }
- }
- },
- "size": {
- "type": "long"
- }
- }
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
deleted file mode 100644
index 3898c8c68..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/propertyType.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ],
- "properties": {
- "dateRanges": {
- "properties": {
- }
- },
- "metadata": {
- "properties": {
- "enabled": {
- "type": "boolean"
- },
- "hidden": {
- "type": "boolean"
- },
- "missingPlugins": {
- "type": "boolean"
- },
- "readOnly": {
- "type": "boolean"
- }
- }
- },
- "multivalued": {
- "type": "boolean"
- },
- "numericRanges": {
- "properties": {
- "from": {
- "type": "double"
- },
- "to": {
- "type": "double"
- }
- }
- },
- "protected": {
- "type": "boolean"
- },
- "rank": {
- "type": "double"
- }
- }
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rulestats.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rulestats.json
deleted file mode 100644
index e1ac5f8d4..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rulestats.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ]
-}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/systemItems.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/systemItems.json
new file mode 100644
index 000000000..ca5a7a397
--- /dev/null
+++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/systemItems.json
@@ -0,0 +1,141 @@
+{
+ "dynamic_templates": [
+ {
+ "all": {
+ "match": "*",
+ "match_mapping_type": "string",
+ "mapping": {
+ "type": "text",
+ "analyzer": "folding",
+ "fields": {
+ "keyword": {
+ "type": "keyword",
+ "ignore_above": 256
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "cost": {
+ "type": "double"
+ },
+ "startDate": {
+ "type": "date"
+ },
+ "endDate": {
+ "type": "date"
+ },
+ "metadata": {
+ "properties": {
+ "enabled": {
+ "type": "boolean"
+ },
+ "hidden": {
+ "type": "boolean"
+ },
+ "missingPlugins": {
+ "type": "boolean"
+ },
+ "readOnly": {
+ "type": "boolean"
+ }
+ }
+ },
+ "entryCondition": {
+ "type": "object",
+ "enabled": false
+ },
+ "parentCondition": {
+ "type": "object",
+ "enabled": false
+ },
+ "startEvent": {
+ "type": "object",
+ "enabled": false
+ },
+ "targetEvent": {
+ "type": "object",
+ "enabled": false
+ },
+ "eventDate": {
+ "type": "date"
+ },
+ "multivalued": {
+ "type": "boolean"
+ },
+ "numericRanges": {
+ "properties": {
+ "from": {
+ "type": "double"
+ },
+ "to": {
+ "type": "double"
+ }
+ }
+ },
+ "protected": {
+ "type": "boolean"
+ },
+ "rank": {
+ "type": "double"
+ },
+ "dateRanges": {
+ "properties": {
+ }
+ },
+ "priority": {
+ "type": "long"
+ },
+ "raiseEventOnlyOnceForProfile": {
+ "type": "boolean"
+ },
+ "raiseEventOnlyOnceForSession": {
+ "type": "boolean"
+ },
+ "raiseEventOnlyOnce": {
+ "type": "boolean"
+ },
+ "condition": {
+ "type": "object",
+ "enabled": false
+ },
+ "actions": {
+ "properties": {
+ "parameterValues": {
+ "type": "object",
+ "enabled": false
+ }
+ }
+ },
+ "elements": {
+ "properties": {
+ "condition": {
+ "type": "object",
+ "enabled": false
+ }
+ }
+ },
+
+ "patchedItemId": {
+ "type": "text"
+ },
+ "patchedItemType": {
+ "type": "text"
+ },
+ "operation": {
+ "type": "text"
+ },
+ "data": {
+ "type": "object",
+ "enabled": false
+ },
+ "lastApplication": {
+ "type": "date"
+ },
+ "schema": {
+ "type": "text"
+ }
+ }
+}
\ No newline at end of file
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/topic.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/topic.json
deleted file mode 100644
index d2d90cb94..000000000
--- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/topic.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "dynamic_templates": [
- {
- "all": {
- "match": "*",
- "match_mapping_type": "string",
- "mapping": {
- "type": "text",
- "analyzer": "folding",
- "fields": {
- "keyword": {
- "type": "keyword",
- "ignore_above": 256
- }
- }
- }
- }
- }
- ]
-}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
index e0ffbee56..fafec7bcf 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
@@ -42,10 +42,12 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
public class RulesServiceImpl implements RulesService, EventListenerService, SynchronousBundleListener {
public static final String RULE_QUERY_PREFIX = "rule_";
+ private static final String RULE_STAT_ID_SUFFIX = "-stat";
public static final String TRACKED_PARAMETER = "trackedConditionParameters";
private static final Logger logger = LoggerFactory.getLogger(RulesServiceImpl.class.getName());
@@ -251,9 +253,10 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
}
private RuleStatistics getLocalRuleStatistics(Rule rule) {
- RuleStatistics ruleStatistics = this.allRuleStatistics.get(rule.getItemId());
+ String ruleStatisticsId = getRuleStatisticId(rule.getItemId());
+ RuleStatistics ruleStatistics = this.allRuleStatistics.get(ruleStatisticsId);
if (ruleStatistics == null) {
- ruleStatistics = new RuleStatistics(rule.getItemId());
+ ruleStatistics = new RuleStatistics(ruleStatisticsId);
}
return ruleStatistics;
}
@@ -264,6 +267,10 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics);
}
+ private String getRuleStatisticId(String ruleID) {
+ return ruleID + RULE_STAT_ID_SUFFIX;
+ }
+
public void refreshRules() {
try {
// we use local variables to make sure we quickly switch the collections since the refresh is called often
@@ -330,21 +337,24 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
RuleStatistics ruleStatistics = getLocalRuleStatistics(rule);
ruleStatistics.setLocalExecutionCount(ruleStatistics.getLocalExecutionCount() + 1);
ruleStatistics.setLocalActionsTime(ruleStatistics.getLocalActionsTime() + totalActionsTime);
- this.allRuleStatistics.put(rule.getItemId(), ruleStatistics);
+ this.allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics);
}
return changes;
}
@Override
public RuleStatistics getRuleStatistics(String ruleId) {
- if (allRuleStatistics.containsKey(ruleId)) {
- return allRuleStatistics.get(ruleId);
+ String ruleStatisticsId = getRuleStatisticId(ruleId);
+ if (allRuleStatistics.containsKey(ruleStatisticsId)) {
+ return allRuleStatistics.get(ruleStatisticsId);
}
- return persistenceService.load(ruleId, RuleStatistics.class);
+ return persistenceService.load(ruleStatisticsId, RuleStatistics.class);
}
public Map<String, RuleStatistics> getAllRuleStatistics() {
- return allRuleStatistics;
+ return allRuleStatistics.keySet().stream()
+ .collect(Collectors.toMap(key -> key.endsWith(RULE_STAT_ID_SUFFIX) ?
+ key.substring(0, key.length() - RULE_STAT_ID_SUFFIX.length()) : key, allRuleStatistics::get));
}
@Override
diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index a12fd6ec7..2050d6ba5 100644
--- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -173,8 +173,8 @@ public class MigrationUtils {
return baseRequest.replace("#rolloverHotActions", rolloverHotActions.toString());
}
- public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName) throws Exception {
- String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName);
+ public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName, String painlessScript) throws Exception {
+ String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName).replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : "");
HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null);
}
@@ -225,10 +225,13 @@ public class MigrationUtils {
HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexNameCloned, null);
}
});
- // Do a refresh
- HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null);
- waitForYellowStatus(httpClient, esAddress, migrationContext);
+ migrationContext.performMigrationStep("Reindex step for: " + indexName + " (refresh at the end)", () -> {
+ // Do a refresh
+ HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName + "/_refresh", null, null);
+
+ waitForYellowStatus(httpClient, esAddress, migrationContext);
+ });
}
public static void scrollQuery(CloseableHttpClient httpClient, String esAddress, String queryURL, String query, String scrollDuration, ScrollCallback scrollCallback) throws IOException {
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
index dd841ff11..3d013b1e4 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-05-globalReindex.groovy
@@ -25,7 +25,7 @@ String indexPrefix = context.getConfigString("indexPrefix")
String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
String[] indicesToReindex = ["segment", "scoring", "campaign", "conditionType", "goal", "patch", "rule"];
indicesToReindex.each { indexToReindex ->
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "${indexToReindex}.json")
+ String mapping = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/mappings/${indexToReindex}.json")
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReindex.toLowerCase()}", newIndexSettings, null, context)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
index d9aa5f889..1f63d1b82 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-20-scopes.groovy
@@ -32,7 +32,7 @@ String scopeIndex = indexPrefix + "-scope"
context.performMigrationStep("2.0.0-create-scope-index", () -> {
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, scopeIndex)) {
String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "scope.json")
+ String mapping = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/mappings/scope.json")
String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseRequest, mapping, context, false)
HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + scopeIndex, newIndexSettings, null)
}
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
index a04a4b097..c692fa7a2 100644
--- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy
@@ -58,7 +58,7 @@ Collections.sort(eventSortedIndices)
context.performMigrationStep("2.2.0-migrate-existing-events", () -> {
MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "event")
eventSortedIndices.each { eventIndex ->
- MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event")
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event", null)
sleep(3000)
}
})
@@ -86,7 +86,7 @@ Collections.sort(sessionSortedIndices)
context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> {
MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "session")
sessionSortedIndices.each { sessionIndex ->
- MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session")
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session", null)
sleep(3000)
}
})
diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
new file mode 100644
index 000000000..8a07a5278
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-05-indicesReduction.groovy
@@ -0,0 +1,78 @@
+import org.apache.unomi.shell.migration.service.MigrationContext
+import org.apache.unomi.shell.migration.utils.HttpUtils
+import org.apache.unomi.shell.migration.utils.MigrationUtils
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+MigrationContext context = migrationContext
+String esAddress = context.getConfigString("esAddress")
+String indexPrefix = context.getConfigString("indexPrefix")
+String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json")
+
+context.performMigrationStep("2.2.0-create-systemItems-index", () -> {
+ if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, "${indexPrefix}-systemitems")) {
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "systemItems.json")
+ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
+ HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/${indexPrefix}-systemitems", newIndexSettings, null)
+ }
+})
+
+def indicesToReduce = [
+ actiontype: "systemitems",
+ campaign: "systemitems",
+ campaignevent: "systemitems",
+ goal: "systemitems",
+ userlist: "systemitems",
+ propertytype: "systemitems",
+ scope: "systemitems",
+ conditiontype: "systemitems",
+ rule: "systemitems",
+ scoring: "systemitems",
+ segment: "systemitems",
+ topic: "systemitems",
+ patch: "systemitems",
+ jsonschema: "systemitems",
+ importconfig: "systemitems",
+ exportconfig: "systemitems",
+ rulestats: "systemitems",
+ groovyaction: "systemitems",
+ persona: "profile",
+ personasession: "session"
+]
+def indicesToSuffixIds = [
+ rulestats: "-stat",
+ groovyaction: "-groovySourceCode"
+]
+indicesToReduce.each { indexToReduce ->
+ context.performMigrationStep("2.2.0-reduce-${indexToReduce.key}", () -> {
+ if (MigrationUtils.indexExists(context.getHttpClient(), esAddress, "${indexPrefix}-${indexToReduce.key}")) {
+ def painless = null
+ // check if we need to update the ids of those items first
+ if (indicesToSuffixIds.containsKey(indexToReduce.key)) {
+ painless = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.2.0/suffix_ids.painless").replace("#ID_SUFFIX", indicesToSuffixIds.get(indexToReduce.key))
+ }
+ // move items
+ MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, "${indexPrefix}-${indexToReduce.key}", "${indexPrefix}-${indexToReduce.value}", painless)
+ MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, "${indexPrefix}-${indexToReduce.key}")
+ HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/${indexPrefix}-${indexToReduce.value}/_refresh", null, null);
+ MigrationUtils.waitForYellowStatus(context.getHttpClient(), esAddress, context);
+ }
+ })
+}
+
+
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/campaign.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/campaign.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/campaign.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/conditionType.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/conditionType.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/conditionType.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/goal.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/goal.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/goal.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/patch.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/patch.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/patch.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/patch.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/rule.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/rule.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/rule.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scope.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scope.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scope.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scope.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scoring.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/scoring.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/scoring.json
diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/segment.json
similarity index 100%
rename from persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/segment.json
rename to tools/shell-commands/src/main/resources/requestBody/2.0.0/mappings/segment.json
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
index ddcb79a5e..589e71084 100644
--- a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json
@@ -4,5 +4,5 @@
},
"dest": {
"index": "#dest"
- }
+ }#painless
}
diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
new file mode 100644
index 000000000..3b63549f6
--- /dev/null
+++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/suffix_ids.painless
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ctx._id = ctx._id + '#ID_SUFFIX';
+ctx._source.itemId = ctx._id;
\ No newline at end of file