You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2020/02/28 13:00:09 UTC
[unomi] 01/01: UNOMI-281 Add lastUpdated system property to
Profiles - This patch adds the lastUpdated system property to profiles -
Added as a system property to avoid any issues with migration. If it
doesn't exist in existing installations it will simply be added the next
time a profile is updated. - Any profile modification (including changing
consents,
adding new segments or scoring plans) will update the last update date on
profiles - This patch also includes some various minor improvements to
loggi [...]
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch UNOMI-281-profile-lastupdated
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 20c3f7c7b230b26d1ffbe024dcc373f481a454e2
Author: Serge Huber <sh...@apache.org>
AuthorDate: Fri Feb 28 13:59:55 2020 +0100
UNOMI-281 Add lastUpdated system property to Profiles
- This patch adds the lastUpdated system property to profiles
- Added as a system property to avoid any issues with migration. If it doesn't exist in existing installations it will simply be added the next time a profile is updated.
- Any profile modification (including changing consents, adding new segments or scoring plans) will update the last update date on profiles
- This patch also includes some various minor improvements to logging, that were used while developing this feature
---
.../main/java/org/apache/unomi/api/Profile.java | 14 +++++
.../apache/unomi/services/UserListServiceImpl.java | 14 +++--
.../unomi/privacy/internal/PrivacyServiceImpl.java | 3 +
.../ElasticSearchPersistenceServiceImpl.java | 14 ++---
.../unomi/persistence/spi/PersistenceService.java | 3 +-
.../actions/MergeProfilesOnPropertyAction.java | 10 ++--
.../unomi/plugins/mail/actions/SendMailAction.java | 5 +-
plugins/pom.xml | 1 +
.../apache/unomi/services/impl/ParserHelper.java | 21 ++++---
.../services/impl/profiles/ProfileServiceImpl.java | 3 +
.../services/impl/segments/SegmentServiceImpl.java | 65 ++++++++++++++++------
.../apache/unomi/shell/commands/ProfileList.java | 10 +++-
12 files changed, 116 insertions(+), 47 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/Profile.java b/api/src/main/java/org/apache/unomi/api/Profile.java
index 69c6c21..90a3a90 100644
--- a/api/src/main/java/org/apache/unomi/api/Profile.java
+++ b/api/src/main/java/org/apache/unomi/api/Profile.java
@@ -132,6 +132,20 @@ public class Profile extends Item {
}
/**
+ * Sets a system property, overwriting an existing one if it existed. This call will also created the system
+ * properties hash map if it didn't exist.
+ * @param key the key for the system property hash map
+ * @param value the value for the system property hash map
+ * @return the previous value object if it existing.
+ */
+ public Object setSystemProperty(String key, Object value) {
+ if (this.systemProperties == null) {
+ this.systemProperties = new LinkedHashMap<>();
+ }
+ return this.systemProperties.put(key, value);
+ }
+
+ /**
* {@inheritDoc}
*
* Note that Profiles are always in the shared system scope ({@link Metadata#SYSTEM_SCOPE}).
diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
index 237a52a..dc3bbc8 100644
--- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
+++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
@@ -25,6 +25,7 @@ import org.apache.unomi.api.services.DefinitionsService;
import org.apache.unomi.lists.UserList;
import org.apache.unomi.persistence.spi.PersistenceService;
+import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -85,14 +86,15 @@ public class UserListServiceImpl implements UserListService {
query.setParameter("propertyValue", listId);
List<Profile> profiles = persistenceService.query(query, null, Profile.class);
- Map<String, Object> profileProps;
+ Map<String, Object> profileSystemProperties;
for (Profile p : profiles) {
- profileProps = p.getSystemProperties();
- if(profileProps != null && profileProps.get("lists") != null) {
- int index = ((List) profileProps.get("lists")).indexOf(listId);
+ profileSystemProperties = p.getSystemProperties();
+ if(profileSystemProperties != null && profileSystemProperties.get("lists") != null) {
+ int index = ((List) profileSystemProperties.get("lists")).indexOf(listId);
if(index != -1){
- ((List) profileProps.get("lists")).remove(index);
- persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileProps);
+ ((List) profileSystemProperties.get("lists")).remove(index);
+ profileSystemProperties.put("lastUpdated", new Date());
+ persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileSystemProperties);
}
}
}
diff --git a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java
index 247a7f4..d4db874 100644
--- a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java
+++ b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java
@@ -88,6 +88,9 @@ public class PrivacyServiceImpl implements PrivacyService {
if (profile == null) {
return false;
}
+ Event profileDeletedEvent = new Event("profileDeleted", null, profile, null, null, profile, new Date());
+ profileDeletedEvent.setPersistent(true);
+ eventService.send(profileDeletedEvent);
// we simply overwrite the existing profile with an empty one.
Profile emptyProfile = new Profile(profileId);
profileService.save(emptyProfile);
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 63ff129..b3fd181 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -785,16 +785,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
logger.error("Failure : cause={} , message={}", failure.getCause(), failure.getMessage());
}
} else {
- logger.info("Update By Query has processed {} in {}.", response.getUpdated(), response.getTook().toString());
+ logger.info("Update with query and script processed {} entries in {}.", response.getUpdated(), response.getTook().toString());
}
if (response.isTimedOut()) {
- logger.error("Update By Query ended with timeout!");
+ logger.error("Update with query and script ended with timeout!");
}
if (response.getVersionConflicts() > 0) {
- logger.warn("Update By Query ended with {} Version Conflicts!", response.getVersionConflicts());
+ logger.warn("Update with query and script ended with {} version conflicts!", response.getVersionConflicts());
}
if (response.getNoops() > 0) {
- logger.warn("Update By Query ended with {} noops!", response.getNoops());
+ logger.warn("Update Bwith query and script ended with {} noops!", response.getNoops());
}
}
return true;
@@ -803,8 +803,6 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
} catch (ScriptException e) {
logger.error("Error in the update script : {}\n{}\n{}", e.getScript(), e.getDetailedMessage(), e.getScriptStack());
throw new Exception("Error in the update script");
- } finally {
- return false;
}
}
}.catchingExecuteInClassLoader(true);
@@ -1901,9 +1899,9 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
public T catchingExecuteInClassLoader(boolean logError, Object... args) {
try {
return executeInClassLoader(timerName, args);
- } catch (Exception e) {
+ } catch (Throwable t) {
if (logError) {
- logger.error("Error while executing in class loader", e);
+ logger.error("Error while executing in class loader", t);
}
}
return null;
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index 0151b9c..7311353 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -116,7 +116,8 @@ public interface PersistenceService {
boolean updateWithScript(String itemId, Date dateHint, Class<?> clazz, String script, Map<String, Object> scriptParams);
/**
- * Updates the items of the specified class by a query with a new property value for the specified property name based on a provided script.
+ * Updates the items of the specified class by a query with a new property value for the specified property name
+ * based on provided scripts and script parameters
*
* @param dateHint a Date helping in identifying where the item is located
* @param clazz the Item subclass of the item to update
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
index 2f4d0cf..8d3b54f 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java
@@ -34,9 +34,7 @@ import org.slf4j.LoggerFactory;
import javax.servlet.ServletResponse;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletResponse;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
public class MergeProfilesOnPropertyAction implements ActionExecutor {
private static final Logger logger = LoggerFactory.getLogger(MergeProfilesOnPropertyAction.class.getName());
@@ -183,7 +181,11 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor {
// we must mark all the profiles that we merged into the master as merged with the master, and they will
// be deleted upon next load
profile.setMergedWith(masterProfileId);
- persistenceService.update(profile.getItemId(), null, Profile.class, "mergedWith", masterProfileId);
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("mergedWith", masterProfile);
+ profile.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profile.getSystemProperties());
+ persistenceService.update(profile.getItemId(), null, Profile.class, sourceMap);
}
}
} catch (Exception e) {
diff --git a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java
index 9e1aadb..8e03175 100644
--- a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java
+++ b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java
@@ -33,6 +33,7 @@ import org.stringtemplate.v4.ST;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@@ -112,7 +113,9 @@ public class SendMailAction implements ActionExecutor {
}
}
- event.getProfile().getSystemProperties().put("notificationAck", profileNotif);
+ event.getProfile().setSystemProperty("notificationAck", profileNotif);
+ event.getProfile().setSystemProperty("lastUpdated", new Date());
+
persistenceService.update(event.getProfile().getItemId(), null, Profile.class, "systemProperties", event.getProfile().getSystemProperties());
ST stringTemplate = new ST(template, '$', '$');
diff --git a/plugins/pom.xml b/plugins/pom.xml
index acca9f6..2cf4bec 100644
--- a/plugins/pom.xml
+++ b/plugins/pom.xml
@@ -38,6 +38,7 @@
<module>hover-event</module>
<module>past-event</module>
<module>tracked-event</module>
+ <module>kafka-injector</module>
</modules>
<dependencies>
diff --git a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java
index 6984e6e..a861f6c 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java
@@ -27,9 +27,7 @@ import org.apache.unomi.api.services.DefinitionsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
/**
* Helper class to resolve condition, action and values types when loading definitions from JSON files
@@ -38,6 +36,9 @@ public class ParserHelper {
private static final Logger logger = LoggerFactory.getLogger(ParserHelper.class);
+ private static final Set<String> unresolvedActionTypes = new HashSet<>();
+ private static final Set<String> unresolvedConditionTypes = new HashSet<>();
+
public static boolean resolveConditionType(final DefinitionsService definitionsService, Condition rootCondition) {
if (rootCondition == null) {
return false;
@@ -49,16 +50,18 @@ public class ParserHelper {
if (condition.getConditionType() == null) {
ConditionType conditionType = definitionsService.getConditionType(condition.getConditionTypeId());
if (conditionType != null) {
+ unresolvedConditionTypes.remove(condition.getConditionTypeId());
condition.setConditionType(conditionType);
} else {
result.add(condition.getConditionTypeId());
+ if (!unresolvedConditionTypes.contains(condition.getConditionTypeId())) {
+ unresolvedConditionTypes.add(condition.getConditionTypeId());
+ logger.warn("Couldn't resolve condition type: " + condition.getConditionTypeId());
+ }
}
}
}
});
- if (!result.isEmpty()) {
- logger.warn("Couldn't resolve condition types : " + result);
- }
return result.isEmpty();
}
@@ -105,9 +108,13 @@ public class ParserHelper {
if (action.getActionType() == null) {
ActionType actionType = definitionsService.getActionType(action.getActionTypeId());
if (actionType != null) {
+ unresolvedActionTypes.remove(action.getActionTypeId());
action.setActionType(actionType);
} else {
- logger.warn("Couldn't resolve action types : " + action.getActionTypeId());
+ if (!unresolvedActionTypes.contains(action.getActionTypeId())) {
+ logger.warn("Couldn't resolve action type : " + action.getActionTypeId());
+ unresolvedActionTypes.add(action.getActionTypeId());
+ }
return false;
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
index 18ffbc3..94fb5c5 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
@@ -521,6 +521,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
if (profile.getItemId() == null) {
return null;
}
+ profile.setSystemProperty("lastUpdated", new Date());
if (persistenceService.save(profile)) {
if (forceRefresh) {
// triggering a load will force an in-place refresh, that may be expensive in performance but will make data immediately available.
@@ -534,6 +535,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
public Profile saveOrMerge(Profile profile) {
Profile previousProfile = persistenceService.load(profile.getItemId(), Profile.class);
+ profile.setSystemProperty("lastUpdated", new Date());
if (previousProfile == null) {
if (persistenceService.save(profile)) {
return profile;
@@ -551,6 +553,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
}
public Persona savePersona(Persona profile) {
+ profile.setSystemProperty("lastUpdated", new Date());
if (persistenceService.load(profile.getItemId(), Persona.class) == null) {
Session session = new PersonaSession(UUID.randomUUID().toString(), profile, new Date());
persistenceService.save(profile);
diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
index 212c1b1..9c362ed 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
@@ -346,10 +346,18 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
segmentCondition.setParameter("propertyValue", segmentId);
List<Profile> previousProfiles = persistenceService.query(segmentCondition, null, Profile.class);
+ long updatedProfileCount = 0;
+ long profileRemovalStartTime = System.currentTimeMillis();
for (Profile profileToRemove : previousProfiles) {
profileToRemove.getSegments().remove(segmentId);
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToRemove.getSegments());
+ profileToRemove.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToRemove.getSystemProperties());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap);
+ updatedProfileCount++;
}
+ logger.info("Removed segment from {} profiles in {} ms", updatedProfileCount, System.currentTimeMillis() - profileRemovalStartTime);
// update impacted segments
for (Segment segment : impactedSegments) {
@@ -786,6 +794,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
Map<String, Object> systemProperties = new HashMap<>();
systemProperties.put("pastEvents", pastEventCounts);
try {
+ systemProperties.put("lastUpdated", new Date());
persistenceService.update(profileId, null, Profile.class, "systemProperties", systemProperties);
} catch (Exception e) {
logger.error("Error updating profile {} past event system properties", profileId, e);
@@ -825,7 +834,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
}
private void updateExistingProfilesForSegment(Segment segment) {
- long t = System.currentTimeMillis();
+ long updateProfilesForSegmentStartTime = System.currentTimeMillis();
Condition segmentCondition = new Condition();
long updatedProfileCount = 0;
@@ -862,32 +871,40 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
PartialList<Profile> profilesToAdd = persistenceService.query(profilesToAddCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m");
while (profilesToAdd.getList().size() > 0) {
- long t2= System.currentTimeMillis();
+ long profilesToAddStartTime = System.currentTimeMillis();
for (Profile profileToAdd : profilesToAdd.getList()) {
profileToAdd.getSegments().add(segment.getItemId());
- persistenceService.update(profileToAdd.getItemId(), null, Profile.class, "segments", profileToAdd.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToAdd.getSegments());
+ profileToAdd.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToAdd.getSystemProperties());
+ persistenceService.update(profileToAdd.getItemId(), null, Profile.class, sourceMap);
Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date());
profileUpdated.setPersistent(false);
eventService.send(profileUpdated);
updatedProfileCount++;
}
- logger.info("{} profiles added in segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - t2);
+ logger.info("{} profiles added to segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - profilesToAddStartTime);
profilesToAdd = persistenceService.continueScrollQuery(Profile.class, profilesToAdd.getScrollIdentifier(), profilesToAdd.getScrollTimeValidity());
if (profilesToAdd == null || profilesToAdd.getList().size() == 0) {
break;
}
}
while (profilesToRemove.getList().size() > 0) {
- long t2= System.currentTimeMillis();
+ long profilesToRemoveStartTime = System.currentTimeMillis();
for (Profile profileToRemove : profilesToRemove.getList()) {
profileToRemove.getSegments().remove(segment.getItemId());
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToRemove.getSegments());
+ profileToRemove.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToRemove.getSystemProperties());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap);
Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
profileUpdated.setPersistent(false);
eventService.send(profileUpdated);
updatedProfileCount++;
}
- logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - t2);
+ logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime );
profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity());
if (profilesToRemove == null || profilesToRemove.getList().size() == 0) {
break;
@@ -897,22 +914,31 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
} else {
PartialList<Profile> profilesToRemove = persistenceService.query(segmentCondition, null, Profile.class, 0, 200, "10m");
while (profilesToRemove.getList().size() > 0) {
+ long profilesToRemoveStartTime = System.currentTimeMillis();
for (Profile profileToRemove : profilesToRemove.getList()) {
profileToRemove.getSegments().remove(segment.getItemId());
- persistenceService.update(profileToRemove.getItemId(), null, Profile.class, "segments", profileToRemove.getSegments());
+ Map<String,Object> sourceMap = new HashMap<>();
+ sourceMap.put("segments", profileToRemove.getSegments());
+ profileToRemove.setSystemProperty("lastUpdated", new Date());
+ sourceMap.put("systemProperties", profileToRemove.getSystemProperties());
+ persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap);
+ Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date());
+ profileUpdated.setPersistent(false);
+ eventService.send(profileUpdated);
updatedProfileCount++;
}
+ logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime);
profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity());
if (profilesToRemove == null || profilesToRemove.getList().size() == 0) {
break;
}
}
}
- logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - t);
+ logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - updateProfilesForSegmentStartTime);
}
private void updateExistingProfilesForScoring(Scoring scoring) {
- long t = System.currentTimeMillis();
+ long startTime = System.currentTimeMillis();
Condition scoringCondition = new Condition();
scoringCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition"));
scoringCondition.setParameter("propertyName", "scores." + scoring.getItemId());
@@ -922,13 +948,17 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
HashMap<String, Object>[] scriptParams = new HashMap[scoring.getElements().size() + 1];
Condition[] conditions = new Condition[scoring.getElements().size() + 1];
+ String lastUpdatedScriptPart = " if (!ctx._source.containsKey(\"systemProperties\")) { ctx._source.put(\"systemProperties\", [:]) } ctx._source.systemProperties.put(\"lastUpdated\", ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of(\"Z\")))";
+
scriptParams[0] = new HashMap<String, Object>();
scriptParams[0].put("scoringId", scoring.getItemId());
- scripts[0] = "if( ctx._source.containsKey(\"systemProperties\") && ctx._source.systemProperties.containsKey(\"scoreModifiers\") && ctx._source.systemProperties.scoreModifiers.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.systemProperties.scoreModifiers.get(params.scoringId)) } else { ctx._source.scores.remove(params.scoringId) }";
+ scripts[0] = "if (ctx._source.containsKey(\"systemProperties\") && ctx._source.systemProperties.containsKey(\"scoreModifiers\") && ctx._source.systemProperties.scoreModifiers.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.systemProperties.scoreModifiers.get(params.scoringId)) } else { ctx._source.scores.remove(params.scoringId) } " +
+ lastUpdatedScriptPart;
conditions[0] = scoringCondition;
if (scoring.getMetadata().isEnabled()) {
- String scriptToAdd = "if( !ctx._source.containsKey(\"scores\") ){ ctx._source.put(\"scores\", [:])} if( ctx._source.scores.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.scores.get(params.scoringId)+params.scoringValue) } else { ctx._source.scores.put(params.scoringId, params.scoringValue) }";
+ String scriptToAdd = "if (!ctx._source.containsKey(\"scores\")) { ctx._source.put(\"scores\", [:])} if (ctx._source.scores.containsKey(params.scoringId) ) { ctx._source.scores.put(params.scoringId, ctx._source.scores.get(params.scoringId)+params.scoringValue) } else { ctx._source.scores.put(params.scoringId, params.scoringValue) } " +
+ lastUpdatedScriptPart;
int idx = 1;
for (ScoringElement element : scoring.getElements()) {
scriptParams[idx] = new HashMap<>();
@@ -939,13 +969,12 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
idx++;
}
}
-
persistenceService.updateWithQueryAndScript(null, Profile.class, scripts, scriptParams, conditions);
- logger.info("Profiles updated in {}ms", System.currentTimeMillis() - t);
+ logger.info("Updated scoring for profiles in {}ms", System.currentTimeMillis() - startTime);
}
private void updateExistingProfilesForRemovedScoring(String scoringId) {
- long t = System.currentTimeMillis();
+ long startTime = System.currentTimeMillis();
Condition scoringCondition = new Condition();
scoringCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition"));
scoringCondition.setParameter("propertyName", "scores." + scoringId);
@@ -958,11 +987,11 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
scriptParams[0].put("scoringId", scoringId);
String[] script = new String[1];
- script[0] = "ctx._source.scores.remove(params.scoringId)";
+ script[0] = "ctx._source.scores.remove(params.scoringId); if (!ctx._source.containsKey(\"systemProperties\")) { ctx._source.put(\"systemProperties\", [:]) } ctx._source.systemProperties.put(\"lastUpdated\", ZonedDateTime.ofInstant(Instant.ofEpochMilli(System.currentTimeMillis()), ZoneId.of(\"Z\")))";
persistenceService.updateWithQueryAndScript(null, Profile.class, script, scriptParams, conditions);
- logger.info("Profiles updated in {}ms", System.currentTimeMillis() - t);
+ logger.info("Removed scoring from profiles in {}ms", System.currentTimeMillis() - startTime);
}
public void bundleChanged(BundleEvent event) {
diff --git a/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java b/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java
index 14f16a9..24f3a2c 100644
--- a/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java
+++ b/tools/shell-dev-commands/src/main/java/org/apache/unomi/shell/commands/ProfileList.java
@@ -51,14 +51,15 @@ public class ProfileList extends ListCommandSupport {
"Scope",
"Segments",
"Consents",
- "Last modification",
+ "Last visit",
+ "Last update"
};
}
@java.lang.Override
protected DataTable buildDataTable() {
Query query = new Query();
- query.setSortby("properties.lastVisit:desc");
+ query.setSortby("systemProperties.lastUpdated:desc,properties.lastVisit:desc");
query.setLimit(maxEntries);
Condition matchAllCondition = new Condition(definitionsService.getConditionType("matchAllCondition"));
query.setCondition(matchAllCondition);
@@ -71,6 +72,11 @@ public class ProfileList extends ListCommandSupport {
rowData.add(StringUtils.join(profile.getSegments(), ","));
rowData.add(StringUtils.join(profile.getConsents().keySet(), ","));
rowData.add((String) profile.getProperty("lastVisit"));
+ if (profile.getSystemProperties() != null && profile.getSystemProperties().get("lastUpdated") != null) {
+ rowData.add((String) profile.getSystemProperties().get("lastUpdated"));
+ } else {
+ rowData.add("");
+ }
dataTable.addRow(rowData.toArray(new Comparable[rowData.size()]));
}
return dataTable;