You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2020/02/28 13:00:08 UTC
[unomi] branch UNOMI-281-profile-lastupdated created (now 20c3f7c)
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a change to branch UNOMI-281-profile-lastupdated
in repository https://gitbox.apache.org/repos/asf/unomi.git.
at 20c3f7c UNOMI-281 Add lastUpdated system property to Profiles - This patch adds the lastUpdated system property to profiles - Added as a system property to avoid any issues with migration. If it doesn't exist in existing installations it will simply be added the next time a profile is updated. - Any profile modification (including changing consents, adding new segments or scoring plans) will update the last update date on profiles - This patch also includes some various minor im [...]
This branch includes the following new commits:
new 20c3f7c UNOMI-281 Add lastUpdated system property to Profiles - This patch adds the lastUpdated system property to profiles - Added as a system property to avoid any issues with migration. If it doesn't exist in existing installations it will simply be added the next time a profile is updated. - Any profile modification (including changing consents, adding new segments or scoring plans) will update the last update date on profiles - This patch also includes some various minor im [...]
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-281 Add lastUpdated system property to
Profiles - This patch adds the lastUpdated system property to profiles -
Added as a system property to avoid any issues with migration. If it
doesn't exist in existing installations it will simply be added the next
time a profile is updated. - Any profile modification (including changing
consents,
adding new segments or scoring plans) will update the last update date on
profiles - This patch also includes some various minor improvements to
loggi [...]
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch UNOMI-281-profile-lastupdated
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 20c3f7c7b230b26d1ffbe024dcc373f481a454e2
Author: Serge Huber <sh...@apache.org>
AuthorDate: Fri Feb 28 13:59:55 2020 +0100
UNOMI-281 Add lastUpdated system property to Profiles
- This patch adds the lastUpdated system property to profiles
- Added as a system property to avoid any issues with migration. If it doesn't exist in existing installations it will simply be added the next time a profile is updated.
- Any profile modification (including changing consents, adding new segments or scoring plans) will update the last update date on profiles
- This patch also includes some various minor improvements to logging, that were used while developing this feature
---
.../main/java/org/apache/unomi/api/Profile.java | 14 +++++
.../apache/unomi/services/UserListServiceImpl.java | 14 +++--
.../unomi/privacy/internal/PrivacyServiceImpl.java | 3 +
.../ElasticSearchPersistenceServiceImpl.java | 14 ++---
.../unomi/persistence/spi/PersistenceService.java | 3 +-
.../actions/MergeProfilesOnPropertyAction.java | 10 ++--
.../unomi/plugins/mail/actions/SendMailAction.java | 5 +-
plugins/pom.xml | 1 +
.../apache/unomi/services/impl/ParserHelper.java | 21 ++++---
.../services/impl/profiles/ProfileServiceImpl.java | 3 +
.../services/impl/segments/SegmentServiceImpl.java | 65 ++++++++++++++++------
.../apache/unomi/shell/commands/ProfileList.java | 10 +++-
12 files changed, 116 insertions(+), 47 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/Profile.java b/api/src/main/java/org/apache/unomi/api/Profile.java
index 69c6c21..90a3a90 100644
--- a/api/src/main/java/org/apache/unomi/api/Profile.java
+++ b/api/src/main/java/org/apache/unomi/api/Profile.java
@@ -132,6 +132,20 @@ public class Profile extends Item {
}
/**
+ * Sets a system property, overwriting an existing one if it existed. This call will also created the system
+ * properties hash map if it didn't exist.
+ * @param key the key for the system property hash map
+ * @param value the value for the system property hash map
+ * @return the previous value object if it existing.
+ */
+ public Object setSystemProperty(String key, Object value) {
+ if (this.systemProperties == null) {
+ this.systemProperties = new LinkedHashMap<>();
+ }
+ return this.systemProperties.put(key, value);
+ }
+
+ /**
* {@inheritDoc}
*
* Note that Profiles are always in the shared system scope ({@link Metadata#SYSTEM_SCOPE}).
diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
index 237a52a..dc3bbc8 100644
--- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
+++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
@@ -25,6 +25,7 @@ import org.apache.unomi.api.services.DefinitionsService;
import org.apache.unomi.lists.UserList;
import org.apache.unomi.persistence.spi.PersistenceService;
+import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -85,14 +86,15 @@ public class UserListServiceImpl implements UserListService {
query.setParameter("propertyValue", listId);
List<Profile> profiles = persistenceService.query(query, null, Profile.class);
- Map<String, Object> profileProps;
+ Map<String, Object> profileSystemProperties;
for (Profile p : profiles) {
- profileProps = p.getSystemProperties();
- if(profileProps != null && profileProps.get("lists") != null) {
- int index = ((List) profileProps.get("lists")).indexOf(listId);
+ profileSystemProperties = p.getSystemProperties();
+ if(profileSystemProperties != null && profileSystemProperties.get("lists") != null) {
+ int index = ((List) profileSystemProperties.get("lists")).indexOf(listId);
if(index != -1){
- ((List) profileProps.get("lists")).remove(index);
- persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileProps);
+ ((List) profileSystemProperties.get("lists")).remove(index);
+ profileSystemProperties.put("lastUpdated", new Date());
+ persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileSystemProperties);
}
}
}
diff --git a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java
index 247a7f4..d4db874 100644
--- a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java
+++ b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java
@@ -88,6 +88,9 @@ public class PrivacyServiceImpl implements PrivacyService {
if (profile == null) {
return false;
}
+ Event profileDeletedEvent = new Event("profileDeleted", null, profile, null, null, profile, new Date());
+ profileDeletedEvent.setPersistent(true);
+ eventService.send(profileDeletedEvent);
// we simply overwrite the existing profile with an empty one.
Profile emptyProfile = new Profile(profileId);
profileService.save(emptyProfile);
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 63ff129..b3fd181 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -785,16 +785,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.error("Failure : cause={} , message={}", failure.getCause(), failure.getMessage());
}
} else {
- logger.info("Update By Query has processed {} in {}.", response.getUpdated(), response.getTook().toString());
+ logger.info("Update with query and script processed {} entries in {}.", response.getUpdated(), response.getTook().toString());
}
if (response.isTimedOut()) {
- logger.error("Update By Query ended with timeout!");
+ logger.error("Update with query and script ended with timeout!");
}
if (response.getVersionConflicts() > 0) {
- logger.warn("Update By Query ended with {} Version Conflicts!", response.getVersionConflicts());
+ logger.warn("Update with query and script ended with {} version conflicts!", response.getVersionConflicts());
}
if (response.getNoops() > 0) {
- logger.warn("Update By Query ended with {} noops!", response.getNoops());
+ logger.warn("Update Bwith query and script ended with {} noops!", response.getNoops());
}
}
return true;
@@ -803,8 +803,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
} catch (ScriptException e) {
logger.error("Error in the update script : {}\n{}\n{}", e.getScript(), e.getDetailedMessage(), e.getScriptStack());
throw new Exception("Error in the update script");
- } finally {
- return false;
}
}
}.catchingExecuteInClassLoader(true);
@@ -1901,9 +1899,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public T catchingExecuteInClassLoader(boolean logError, Object... args) {
try {
return executeInClassLoader(timerName, args);
- } catch (Exception e) {
+ } catch (Throwable t) {
if (logError) {
- logger.error("Error while executing in class loader", e);
+ logger.error("Error while executing in class loader", t);
}
}
return null;
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 0151b9c..7311353 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -116,7 +116,8 @@ public interface PersistenceService {
boolean updateWithScript(String itemId, Date dateHint, Class<?> clazz, String script, Map<String, Object> scriptParams);
/**
- * Updates the items of the specified class by a query with a new property value for the specified property name based on a provided script.
+ * Updates the items of the specified class by a query with a new property value for the specified property name
+ * based on provided scripts and script parameters
*
* @param dateHint a Date helping in identifying where the item is located
* @param clazz the Item subclass of the item to update
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
index 2f4d0cf..8d3b54f 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
@@ -34,9 +34,7 @@ import org.slf4j.LoggerFactory;
import javax.servlet.ServletResponse;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletResponse;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
public class MergeProfilesOnPropertyAction implements ActionExecutor {
private static final Logger logger = LoggerFactory.getLogger(MergeProfilesOnPropertyAction.class.getName());
@@ -183,7 +181,11 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
// we must mark all the profiles that we merged into the master as merged with the master, and they will
// be deleted upon next load
profile.setMergedWith(masterProfileId);
- persistenceService.update(profile.getItemId(), null, Profile.class, "mergedWith", masterProfileId);
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("mergedWith", masterProfile);
+ profile.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profile.getSystemProperties());
+ persistenceService.update(profile.getItemId(), null, Profile.class, sourceMap);
}
}
} catch (Exception e) {
diff --git a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java
index 9e1aadb..8e03175 100644
--- a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java
+++ b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java
@@ -33,6 +33,7 @@ import org.stringtemplate.v4.ST;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@@ -112,7 +113,9 @@ public class SendMailAction implements ActionExecutor {
}
}
- event.getProfile().getSystemProperties().put("notificationAck", profileNotif);
+ event.getProfile().setSystemProperty("notificationAck", profileNotif);
+ event.getProfile().setSystemProperty("lastUpdated", new Date());
+
persistenceService.update(event.getProfile().getItemId(), null, Profile.class, "systemProperties", event.getProfile().getSystemProperties());
ST stringTemplate = new ST(template, '$', '$');
diff --git a/plugins/pom.xml b/plugins/pom.xml
index acca9f6..2cf4bec 100644
--- a/plugins/pom.xml
+++ b/plugins/pom.xml
@@ -38,6 +38,7 @@
<module>hover-event</module>
<module>past-event</module>
<module>tracked-event</module>
+ <module>kafka-injector</module>
</modules>
<dependencies>
diff --git a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java
index 6984e6e..a861f6c 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java
@@ -27,9 +27,7 @@ import org.apache.unomi.api.services.DefinitionsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
/**
* Helper class to resolve condition, action and values types when loading definitions from JSON files
@@ -38,6 +36,9 @@ public class ParserHelper {
private static final Logger logger = LoggerFactory.getLogger(ParserHelper.class);
+ private static final Set<String> unresolvedActionTypes = new HashSet<>();
+ private static final Set<String> unresolvedConditionTypes = new HashSet<>();
+
public static boolean resolveConditionType(final DefinitionsService definitionsService, Condition rootCondition) {
if (rootCondition == null) {
return false;
@@ -49,16 +50,18 @@ public class ParserHelper {
if (condition.getConditionType() == null) {
ConditionType conditionType = definitionsService.getConditionType(condition.getConditionTypeId());
if (conditionType != null) {
+ unresolvedConditionTypes.remove(condition.getConditionTypeId());
condition.setConditionType(conditionType);
} else {
result.add(condition.getConditionTypeId());
+ if (!unresolvedConditionTypes.contains(condition.getConditionTypeId())) {
+ unresolvedConditionTypes.add(condition.getConditionTypeId());
+ logger.warn("Couldn't resolve condition type: " + condition.getConditionTypeId());
+ }
}
}
}
});
- if (!result.isEmpty()) {
- logger.warn("Couldn't resolve condition types : " + result);
- }
return result.isEmpty();
}
@@ -105,9 +108,13 @@ public class ParserHelper {
if (action.getActionType() == null) {
ActionType actionType = definitionsService.getActionType(action.getActionTypeId());
if (actionType != null) {
+ unresolvedActionTypes.remove(action.getActionTypeId());
action.setActionType(actionType);
} else {
- logger.warn("Couldn't resolve action types : " + action.getActionTypeId());
+ if (!unresolvedActionTypes.contains(action.getActionTypeId())) {
+ logger.warn("Couldn't resolve action type : " + action.getActionTypeId());
+ unresolvedActionTypes.add(action.getActionTypeId());
+ }
return false;
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
index 18ffbc3..94fb5c5 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
@@ -521,6 +521,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
if (profile.getItemId() == null) {
return null;
}
+ profile.setSystemProperty("lastUpdated", new Date());
if (persistenceService.save(profile)) {
if (forceRefresh) {
// triggering a load will force an in-place refresh, that may be expensive in performance but will make data immediately available.
@@ -534,6 +535,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
public Profile saveOrMerge(Profile profile) {
Profile previousProfile = persistenceService.load(profile.getItemId(), Profile.class);
+ profile.setSystemProperty("lastUpdated", new Date());
if (previousProfile == null) {
if (persistenceService.save(profile)) {
return profile;
@@ -551,6 +553,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
}
public Persona savePersona(Persona profile) {
+ profile.setSystemProperty("lastUpdated", new Date());
if (persistenceService.load(profile.getItemId(), Persona.class) == null) {
Session session = new PersonaSession(UUID.randomUUID().toString(), profile, new Date());
persistenceService.save(profile);
diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
index 212c1b1..9c362ed 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
@@ -346,10 +346,18 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
segmentCondition.setParameter("propertyValue", segmentId);
List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class);
+ long updatedProfileCount = 0;
+ long profileRemovalStartTime = System.currentTimeMillis();
for (Profile profileToRemove : previousProfiles) {
profileToRemove.getSegments().remove(segmentId);
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToRemove.getSegments());
+ profileToRemove.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToRemove.getSystemProperties());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap);
+ updatedProfileCount++;
}
+ logger.info("Removed segment from {} profiles in {} ms", updatedProfileCount, System.currentTimeMillis() - profileRemovalStartTime);
// update impacted segments
for (Segment segment : impactedSegments) {
@@ -786,6 +794,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
Map<String, Object> systemProperties = new HashMap<>();
systemProperties.put("pastEvents", pastEventCounts);
try {
+ systemProperties.put("lastUpdated", new Date());
persistenceService.update(profileId, null, Profile.class, "systemProperties", systemProperties);
} catch (Exception e) {
logger.error("Error updating profile {} past event system properties", profileId, e);
@@ -825,7 +834,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
}
private void updateExistingProfilesForSegment(Segment segment) {
- long t = System.currentTimeMillis();
+ long updateProfilesForSegmentStartTime = System.currentTimeMillis();
Condition segmentCondition = new Condition();
long updatedProfileCount = 0;
@@ -862,32 +871,40 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
PartialList<Profile> profilesToAdd = persistenceService.query(profilesToAddCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m");
while (profilesToAdd.getList().size() > 0) {
- long t2= System.currentTimeMillis();
+ long profilesToAddStartTime = System.currentTimeMillis();
for (Profile profileToAdd : profilesToAdd.getList()) {
profileToAdd.getSegments().add(segment.getItemId());
- persistenceService.update(profileToAdd.getItemId(), null, Profile.class, "segments", profileToAdd.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToAdd.getSegments());
+ profileToAdd.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToAdd.getSystemProperties());
+ persistenceService.update(profileToAdd.getItemId(), null, Profile.class, sourceMap);
Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date());
profileUpdated.setPersistent(false);
eventService.send(profileUpdated);
updatedProfileCount++;
}
- logger.info("{} profiles added in segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - t2);
+ logger.info("{} profiles added to segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - profilesToAddStartTime);
profilesToAdd = persistenceService.continueScrollQuery(Profile.class, profilesToAdd.getScrollIdentifier(), profilesToAdd.getScrollTimeValidity());
if (profilesToAdd == null || profilesToAdd.getList().size() == 0) {
break;
}
}
while (profilesToRemove.getList().size() > 0) {
- long t2= System.currentTimeMillis();
+ long profilesToRemoveStartTime = System.currentTimeMillis();
for (Profile profileToRemove : profilesToRemove.getList()) {
profileToRemove.getSegments().remove(segment.getItemId());
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToRemove.getSegments());
+ profileToRemove.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToRemove.getSystemProperties());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap);
Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
profileUpdated.setPersistent(false);
eventService.send(profileUpdated);
updatedProfileCount++;
}
- logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - t2);
+ logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime );
profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity());
if (profilesToRemove == null || profilesToRemove.getList().size() == 0) {
break;
@@ -897,22 +914,31 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
} else {
PartialList<Profile> profilesToRemove = persistenceService.query(segmentCondition, null, Profile.class, 0, 200, "10m");
while (profilesToRemove.getList().size() > 0) {
+ long profilesToRemoveStartTime = System.currentTimeMillis();
for (Profile profileToRemove : profilesToRemove.getList()) {
profileToRemove.getSegments().remove(segment.getItemId());
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToRemove.getSegments());
+ profileToRemove.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToRemove.getSystemProperties());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap);
+ Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
+ profileUpdated.setPersistent(false);
+ eventService.send(profileUpdated);
updatedProfileCount++;
}
+ logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime);
profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity());
if (profilesToRemove == null || profilesToRemove.getList().size() == 0) {
break;
}
}
}
- logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - t);
+ logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - updateProfilesForSegmentStartTime);
}
private void updateExistingProfilesForScoring(Scoring scoring) {
- long t = System.currentTimeMillis();
+ long startTime = System.currentTimeMillis();
Condition scoringCondition = new Condition();
scoringCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition"));
scoringCondition.setParameter("propertyName", "scores." + scoring.getItemId());
@@ -922,13 +948,17 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
HashMap<String, Object>[] scriptParams = new HashMap[scoring.getElements().size() + 1];
Condition[] conditions = new Condition[scoring.getElements().size() + 1];
+ String lastUpdatedScriptPart = " if (!ctx._source.containsKey(\"systemProperties\")) { ctx._source.put(\"systemProperties\", [:]) } ctx._source.systemProperties.put(\"lastUpdated\", ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of(\"Z\")))";
+
scriptParams[0] = new HashMap<String, Object>();
scriptParams[0].put("scoringId", scoring.getItemId());
- scripts[0] = "if( ctx._source.containsKey(\"systemProperties\") && ctx._source.systemProperties.containsKey(\"scoreModifiers\") && ctx._source.systemProperties.scoreModifiers.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.systemProperties.scoreModifiers.get(params.scoringId)) } else { ctx._source.scores.remove(params.scoringId) }";
+ scripts[0] = "if (ctx._source.containsKey(\"systemProperties\") && ctx._source.systemProperties.containsKey(\"scoreModifiers\") && ctx._source.systemProperties.scoreModifiers.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.systemProperties.scoreModifiers.get(params.scoringId)) } else { ctx._source.scores.remove(params.scoringId) } " +
+ lastUpdatedScriptPart;
conditions[0] = scoringCondition;
if (scoring.getMetadata().isEnabled()) {
- String scriptToAdd = "if( !ctx._source.containsKey(\"scores\") ){ ctx._source.put(\"scores\", [:])} if( ctx._source.scores.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.scores.get(params.scoringId)+params.scoringValue) } else { ctx._source.scores.put(params.scoringId, params.scoringValue) }";
+ String scriptToAdd = "if (!ctx._source.containsKey(\"scores\")) { ctx._source.put(\"scores\", [:])} if (ctx._source.scores.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.scores.get(params.scoringId)+params.scoringValue) } else { ctx._source.scores.put(params.scoringId, params.scoringValue) } " +
+ lastUpdatedScriptPart;
int idx = 1;
for (ScoringElement element : scoring.getElements()) {
scriptParams[idx] = new HashMap<>();
@@ -939,13 +969,12 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
idx++;
}
}
-
persistenceService.updateWithQueryAndScript(null, Profile.class, scripts, scriptParams, conditions);
- logger.info("Profiles updated in {}ms", System.currentTimeMillis() - t);
+ logger.info("Updated scoring for profiles in {}ms", System.currentTimeMillis() - startTime);
}
private void updateExistingProfilesForRemovedScoring(String scoringId) {
- long t = System.currentTimeMillis();
+ long startTime = System.currentTimeMillis();
Condition scoringCondition = new Condition();
scoringCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition"));
scoringCondition.setParameter("propertyName", "scores." + scoringId);
@@ -958,11 +987,11 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
scriptParams[0].put("scoringId", scoringId);
String[] script = new String[1];
- script[0] = "ctx._source.scores.remove(params.scoringId)";
+ script[0] = "ctx._source.scores.remove(params.scoringId); if (!ctx._source.containsKey(\"systemProperties\")) { ctx._source.put(\"systemProperties\", [:]) } ctx._source.systemProperties.put(\"lastUpdated\", ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of(\"Z\")))";
persistenceService.updateWithQueryAndScript(null, Profile.class, script, scriptParams, conditions);
- logger.info("Profiles updated in {}ms", System.currentTimeMillis() - t);
+ logger.info("Removed scoring from profiles in {}ms", System.currentTimeMillis() - startTime);
}
public void bundleChanged(BundleEvent event) {
diff --git a/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java b/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java
index 14f16a9..24f3a2c 100644
--- a/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java
+++ b/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java
@@ -51,14 +51,15 @@ public class ProfileList extends ListCommandSupport {
"Scope",
"Segments",
"Consents",
- "Last modification",
+ "Last visit",
+ "Last update"
};
}
@java.lang.Override
protected DataTable buildDataTable() {
Query query = new Query();
- query.setSortby("properties.lastVisit:desc");
+ query.setSortby("systemProperties.lastUpdated:desc,properties.lastVisit:desc");
query.setLimit(maxEntries);
Condition matchAllCondition = new Condition(definitionsService.getConditionType("matchAllCondition"));
query.setCondition(matchAllCondition);
@@ -71,6 +72,11 @@ public class ProfileList extends ListCommandSupport {
rowData.add(StringUtils.join(profile.getSegments(), ","));
rowData.add(StringUtils.join(profile.getConsents().keySet(), ","));
rowData.add((String) profile.getProperty("lastVisit"));
+ if (profile.getSystemProperties() != null && profile.getSystemProperties().get("lastUpdated") != null) {
+ rowData.add((String) profile.getSystemProperties().get("lastUpdated"));
+ } else {
+ rowData.add("");
+ }
dataTable.addRow(rowData.toArray(new Comparable[rowData.size()]));
}
return dataTable;