You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2023/03/01 17:03:29 UTC
[unomi] 01/01: feedbacks
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch UNOMI-739-update-purge-system
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 1f90b80da0ca7100b39dbfc26f7f19e12d5e5d13
Author: jsinovassin <js...@jahia.com>
AuthorDate: Wed Mar 1 16:21:07 2023 +0100
feedbacks
---
.../apache/unomi/api/services/ProfileService.java | 11 ++++-
.../org/apache/unomi/itests/ProfileServiceIT.java | 23 +++++-----
.../main/resources/etc/custom.system.properties | 4 +-
.../services/impl/profiles/ProfileServiceImpl.java | 51 +++++++++++++---------
.../main/resources/org.apache.unomi.services.cfg | 4 +-
5 files changed, 57 insertions(+), 36 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/services/ProfileService.java b/api/src/main/java/org/apache/unomi/api/services/ProfileService.java
index a7a3e2a3f..03da6ce9b 100644
--- a/api/src/main/java/org/apache/unomi/api/services/ProfileService.java
+++ b/api/src/main/java/org/apache/unomi/api/services/ProfileService.java
@@ -427,13 +427,20 @@ public interface ProfileService {
/**
* Purge (delete) session items
- * @param existsNumberOfDays used to remove monthly indices older than this number of days
+ * @param existsNumberOfDays will purge sessions created since this number of days (0 or negative value, will have no effect)
*/
void purgeSessionItems(int existsNumberOfDays);
/**
* Purge (delete) event items
- * @param existsNumberOfDays used to remove monthly indices older than this number of days
+ * @param existsNumberOfDays will purge events created since this number of days (0 or negative value, will have no effect)
*/
void purgeEventItems(int existsNumberOfDays);
+
+ /**
+ * Use purgeSessionItems and purgeEventItems to remove rollover items instead
+ * @param existsNumberOfMonths used to remove monthly indices older than this number of months
+ */
+ @Deprecated
+ void purgeMonthlyItems(int existsNumberOfMonths);
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
index bda614d81..455ab9073 100644
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
@@ -379,29 +379,30 @@ public class ProfileServiceIT extends BaseIT {
keepTrying("Events number should be 450", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (450 + originalEventsCount), 1000, 100);
- // Should have no effect there is no monthly items older than 40 months
- profileService.purgeSessionItems(40);
+ // Should have no effect there is no sessions items older than 1200 days
+ profileService.purgeSessionItems(1200);
keepTrying("Sessions number should be 450", () -> persistenceService.getAllItemsCount(Session.ITEM_TYPE),
(count) -> count == (450 + originalSessionsCount), 1000, 100);
- profileService.purgeEventItems(40);
+ // Should have no effect there is no events items older than 1200 days
+ profileService.purgeEventItems(1200);
keepTrying("Events number should be 450", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (450 + originalEventsCount), 1000, 100);
- // Should purge sessions older than 25 days
- profileService.purgeSessionItems(25);
+ // Should purge sessions older than 750 days
+ profileService.purgeSessionItems(750);
keepTrying("Sessions number should be 300", () -> persistenceService.getAllItemsCount(Session.ITEM_TYPE),
(count) -> count == (300 + originalSessionsCount), 1000, 100);
- // Should purge events older than 25 days
- profileService.purgeEventItems(25);
+ // Should purge events older than 750 days
+ profileService.purgeEventItems(750);
keepTrying("Events number should be 300", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (300 + originalEventsCount), 1000, 100);
- // Should purge sessions older than 5 days
- profileService.purgeSessionItems(5);
+ // Should purge sessions older than 150 days
+ profileService.purgeSessionItems(150);
keepTrying("Sessions number should be 150", () -> persistenceService.getAllItemsCount(Session.ITEM_TYPE),
(count) -> count == (150 + originalSessionsCount), 1000, 100);
- // Should purge events older than 5 days
- profileService.purgeEventItems(5);
+ // Should purge events older than 150 days
+ profileService.purgeEventItems(150);
keepTrying("Events number should be 150", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (150 + originalEventsCount), 1000, 100);
}
diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties
index 59f187983..68c4a9896 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -162,7 +162,9 @@ org.apache.unomi.profile.purge.interval=${env:UNOMI_PROFILE_PURGE_INTERVAL:-1}
org.apache.unomi.profile.purge.inactiveTime=${env:UNOMI_PROFILE_PURGE_INACTIVETIME:-180}
# Purge profiles that have been created for a specific number of days
org.apache.unomi.profile.purge.existTime=${env:UNOMI_PROFILE_PURGE_EXISTTIME:--1}
-# Purge all monthly indexes (sessions/events) that have been created for a specific number of months
+# Deprecated: Use org.apache.unomi.sessions.purge.existTime and org.apache.unomi.events.purge.existTime instead.
+# If there is no value for org.apache.unomi.sessions.purge.existTime or org.apache.unomi.events.purge.existTime, the value
+# of this property will be use for the purge.
org.apache.unomi.monthly.index.purge.existTime=${env:UNOMI_MONTHLY_INDEX_PURGE_EXISTTIME:-12}
# Purge sessions that have been created for a specific number of days
org.apache.unomi.sessions.purge.existTime=${env:UNOMI_SESSIONS_PURGE_EXISTTIME:-}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
index 05ffb72fd..7912cc6de 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
@@ -370,10 +370,20 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
}
}
- @Override
- public void purgeSessionItems(int existsNumberOfDays) {
+ private <T extends Item> void purgeRolloverItems(int existsNumberOfDays, Class<T> clazz) {
if (existsNumberOfDays > 0) {
- ConditionType propertyConditionType = definitionsService.getConditionType("sessionPropertyCondition");
+ String conditionType = null;
+ String itemType = null;
+
+ if (clazz.getName().equals(Event.class.getName())) {
+ conditionType = "eventPropertyCondition";
+ itemType = Event.ITEM_TYPE;
+ } else if (clazz.getName().equals(Session.class.getName())) {
+ conditionType = "sessionPropertyCondition";
+ itemType = Session.ITEM_TYPE;
+ }
+
+ ConditionType propertyConditionType = definitionsService.getConditionType(conditionType);
if (propertyConditionType == null) {
// definition service not yet fully instantiate
return;
@@ -381,35 +391,34 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
Condition condition = new Condition(propertyConditionType);
- logger.info("Purging: Session created since more than {} days", existsNumberOfDays);
condition.setParameter("propertyName", "timeStamp");
condition.setParameter("comparisonOperator", "lessThanOrEqualTo");
condition.setParameter("propertyValueDateExpr", "now-" + existsNumberOfDays + "d");
+ persistenceService.removeByQuery(condition, clazz);
+ deleteEmptyRolloverIndex(itemType);
+ }
+ }
- persistenceService.removeByQuery(condition, Session.class);
- deleteEmptyRolloverIndex(Session.ITEM_TYPE);
+ @Override
+ public void purgeSessionItems(int existsNumberOfDays) {
+ if (existsNumberOfDays > 0) {
+ logger.info("Purging: Sessions created since more than {} days", existsNumberOfDays);
+ purgeRolloverItems(existsNumberOfDays, Session.class);
}
}
@Override
public void purgeEventItems(int existsNumberOfDays) {
if (existsNumberOfDays > 0) {
- ConditionType propertyConditionType = definitionsService.getConditionType("eventPropertyCondition");
- if (propertyConditionType == null) {
- // definition service not yet fully instantiate
- return;
- }
-
- Condition condition = new Condition(propertyConditionType);
+ logger.info("Purging: Events created since more than {} days", existsNumberOfDays);
+ purgeRolloverItems(existsNumberOfDays, Event.class);
+ }
+ }
- logger.info("Purging: Session created since more than {} days", existsNumberOfDays);
- condition.setParameter("propertyName", "timeStamp");
- condition.setParameter("comparisonOperator", "lessThanOrEqualTo");
- condition.setParameter("propertyValueDateExpr", "now-" + existsNumberOfDays + "d");
+ @Deprecated
+ @Override
+ public void purgeMonthlyItems(int existsNumberOfMonths) {
- persistenceService.removeByQuery(condition, Event.class);
- deleteEmptyRolloverIndex(Event.ITEM_TYPE);
- }
}
public void deleteEmptyRolloverIndex(String indexName) {
@@ -428,7 +437,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
private void initializePurge() {
logger.info("Purge: Initializing");
- if (purgeProfileInactiveTime > 0 || purgeProfileExistTime > 0 || purgeSessionsAndEventsTime > 0 || purgeSessionExistTime > 0 || purgeEventExistTime > 0) {
+ if (purgeProfileInactiveTime > 0 || purgeProfileExistTime > 0 || purgeSessionExistTime > 0 || purgeEventExistTime > 0) {
if (purgeProfileInactiveTime > 0) {
logger.info("Purge: Profile with no visits since more than {} days, will be purged", purgeProfileInactiveTime);
}
diff --git a/services/src/main/resources/org.apache.unomi.services.cfg b/services/src/main/resources/org.apache.unomi.services.cfg
index fbd9dff7d..9cab89153 100644
--- a/services/src/main/resources/org.apache.unomi.services.cfg
+++ b/services/src/main/resources/org.apache.unomi.services.cfg
@@ -27,7 +27,9 @@ profile.purge.existTime=${org.apache.unomi.profile.purge.existTime:--1}
# Refresh Elasticsearch after saving a profile
profile.forceRefreshOnSave=${org.apache.unomi.profile.forceRefreshOnSave:-false}
-# Purge all monthly indexes (sessions/events) that have been created for a specific number of months
+# Deprecated: Use session.purge.existTime and event.purge.existTime instead.
+# If there is no value for session.purge.existTime or event.purge.existTime, the value
+# of this property will be use for the purge.
monthly.index.purge.existTime=${org.apache.unomi.monthly.index.purge.existTime:-12}
# Purge sessions that have been created for a specific number of days