You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by js...@apache.org on 2023/03/01 17:03:28 UTC
[unomi] branch UNOMI-739-update-purge-system updated (be4a5255e -> 1f90b80da)
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a change to branch UNOMI-739-update-purge-system
in repository https://gitbox.apache.org/repos/asf/unomi.git
omit be4a5255e feedbacks
new 1f90b80da feedbacks
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (be4a5255e)
\
N -- N -- N refs/heads/UNOMI-739-update-purge-system (1f90b80da)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[unomi] 01/01: feedbacks
Posted by js...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch UNOMI-739-update-purge-system
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 1f90b80da0ca7100b39dbfc26f7f19e12d5e5d13
Author: jsinovassin <js...@jahia.com>
AuthorDate: Wed Mar 1 16:21:07 2023 +0100
feedbacks
---
.../apache/unomi/api/services/ProfileService.java | 11 ++++-
.../org/apache/unomi/itests/ProfileServiceIT.java | 23 +++++-----
.../main/resources/etc/custom.system.properties | 4 +-
.../services/impl/profiles/ProfileServiceImpl.java | 51 +++++++++++++---------
.../main/resources/org.apache.unomi.services.cfg | 4 +-
5 files changed, 57 insertions(+), 36 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/services/ProfileService.java b/api/src/main/java/org/apache/unomi/api/services/ProfileService.java
index a7a3e2a3f..03da6ce9b 100644
--- a/api/src/main/java/org/apache/unomi/api/services/ProfileService.java
+++ b/api/src/main/java/org/apache/unomi/api/services/ProfileService.java
@@ -427,13 +427,20 @@ public interface ProfileService {
/**
* Purge (delete) session items
- * @param existsNumberOfDays used to remove monthly indices older than this number of days
+ * @param existsNumberOfDays will purge sessions created since this number of days (0 or negative value, will have no effect)
*/
void purgeSessionItems(int existsNumberOfDays);
/**
* Purge (delete) event items
- * @param existsNumberOfDays used to remove monthly indices older than this number of days
+ * @param existsNumberOfDays will purge events created since this number of days (0 or negative value, will have no effect)
*/
void purgeEventItems(int existsNumberOfDays);
+
+ /**
+ * Use purgeSessionItems and purgeEventItems to remove rollover items instead
+ * @param existsNumberOfMonths used to remove monthly indices older than this number of months
+ */
+ @Deprecated
+ void purgeMonthlyItems(int existsNumberOfMonths);
}
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
index bda614d81..455ab9073 100644
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
@@ -379,29 +379,30 @@ public class ProfileServiceIT extends BaseIT {
keepTrying("Events number should be 450", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (450 + originalEventsCount), 1000, 100);
- // Should have no effect there is no monthly items older than 40 months
- profileService.purgeSessionItems(40);
+ // Should have no effect there is no sessions items older than 1200 days
+ profileService.purgeSessionItems(1200);
keepTrying("Sessions number should be 450", () -> persistenceService.getAllItemsCount(Session.ITEM_TYPE),
(count) -> count == (450 + originalSessionsCount), 1000, 100);
- profileService.purgeEventItems(40);
+ // Should have no effect there is no events items older than 1200 days
+ profileService.purgeEventItems(1200);
keepTrying("Events number should be 450", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (450 + originalEventsCount), 1000, 100);
- // Should purge sessions older than 25 days
- profileService.purgeSessionItems(25);
+ // Should purge sessions older than 750 days
+ profileService.purgeSessionItems(750);
keepTrying("Sessions number should be 300", () -> persistenceService.getAllItemsCount(Session.ITEM_TYPE),
(count) -> count == (300 + originalSessionsCount), 1000, 100);
- // Should purge events older than 25 days
- profileService.purgeEventItems(25);
+ // Should purge events older than 750 days
+ profileService.purgeEventItems(750);
keepTrying("Events number should be 300", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (300 + originalEventsCount), 1000, 100);
- // Should purge sessions older than 5 days
- profileService.purgeSessionItems(5);
+ // Should purge sessions older than 150 days
+ profileService.purgeSessionItems(150);
keepTrying("Sessions number should be 150", () -> persistenceService.getAllItemsCount(Session.ITEM_TYPE),
(count) -> count == (150 + originalSessionsCount), 1000, 100);
- // Should purge events older than 5 days
- profileService.purgeEventItems(5);
+ // Should purge events older than 150 days
+ profileService.purgeEventItems(150);
keepTrying("Events number should be 150", () -> persistenceService.getAllItemsCount(Event.ITEM_TYPE),
(count) -> count == (150 + originalEventsCount), 1000, 100);
}
diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties
index 59f187983..68c4a9896 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -162,7 +162,9 @@ org.apache.unomi.profile.purge.interval=${env:UNOMI_PROFILE_PURGE_INTERVAL:-1}
org.apache.unomi.profile.purge.inactiveTime=${env:UNOMI_PROFILE_PURGE_INACTIVETIME:-180}
# Purge profiles that have been created for a specific number of days
org.apache.unomi.profile.purge.existTime=${env:UNOMI_PROFILE_PURGE_EXISTTIME:--1}
-# Purge all monthly indexes (sessions/events) that have been created for a specific number of months
+# Deprecated: Use org.apache.unomi.sessions.purge.existTime and org.apache.unomi.events.purge.existTime instead.
+# If there is no value for org.apache.unomi.sessions.purge.existTime or org.apache.unomi.events.purge.existTime, the value
+# of this property will be use for the purge.
org.apache.unomi.monthly.index.purge.existTime=${env:UNOMI_MONTHLY_INDEX_PURGE_EXISTTIME:-12}
# Purge sessions that have been created for a specific number of days
org.apache.unomi.sessions.purge.existTime=${env:UNOMI_SESSIONS_PURGE_EXISTTIME:-}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
index 05ffb72fd..7912cc6de 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java
@@ -370,10 +370,20 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
}
}
- @Override
- public void purgeSessionItems(int existsNumberOfDays) {
+ private <T extends Item> void purgeRolloverItems(int existsNumberOfDays, Class<T> clazz) {
if (existsNumberOfDays > 0) {
- ConditionType propertyConditionType = definitionsService.getConditionType("sessionPropertyCondition");
+ String conditionType = null;
+ String itemType = null;
+
+ if (clazz.getName().equals(Event.class.getName())) {
+ conditionType = "eventPropertyCondition";
+ itemType = Event.ITEM_TYPE;
+ } else if (clazz.getName().equals(Session.class.getName())) {
+ conditionType = "sessionPropertyCondition";
+ itemType = Session.ITEM_TYPE;
+ }
+
+ ConditionType propertyConditionType = definitionsService.getConditionType(conditionType);
if (propertyConditionType == null) {
// definition service not yet fully instantiate
return;
@@ -381,35 +391,34 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
Condition condition = new Condition(propertyConditionType);
- logger.info("Purging: Session created since more than {} days", existsNumberOfDays);
condition.setParameter("propertyName", "timeStamp");
condition.setParameter("comparisonOperator", "lessThanOrEqualTo");
condition.setParameter("propertyValueDateExpr", "now-" + existsNumberOfDays + "d");
+ persistenceService.removeByQuery(condition, clazz);
+ deleteEmptyRolloverIndex(itemType);
+ }
+ }
- persistenceService.removeByQuery(condition, Session.class);
- deleteEmptyRolloverIndex(Session.ITEM_TYPE);
+ @Override
+ public void purgeSessionItems(int existsNumberOfDays) {
+ if (existsNumberOfDays > 0) {
+ logger.info("Purging: Sessions created since more than {} days", existsNumberOfDays);
+ purgeRolloverItems(existsNumberOfDays, Session.class);
}
}
@Override
public void purgeEventItems(int existsNumberOfDays) {
if (existsNumberOfDays > 0) {
- ConditionType propertyConditionType = definitionsService.getConditionType("eventPropertyCondition");
- if (propertyConditionType == null) {
- // definition service not yet fully instantiate
- return;
- }
-
- Condition condition = new Condition(propertyConditionType);
+ logger.info("Purging: Events created since more than {} days", existsNumberOfDays);
+ purgeRolloverItems(existsNumberOfDays, Event.class);
+ }
+ }
- logger.info("Purging: Session created since more than {} days", existsNumberOfDays);
- condition.setParameter("propertyName", "timeStamp");
- condition.setParameter("comparisonOperator", "lessThanOrEqualTo");
- condition.setParameter("propertyValueDateExpr", "now-" + existsNumberOfDays + "d");
+ @Deprecated
+ @Override
+ public void purgeMonthlyItems(int existsNumberOfMonths) {
- persistenceService.removeByQuery(condition, Event.class);
- deleteEmptyRolloverIndex(Event.ITEM_TYPE);
- }
}
public void deleteEmptyRolloverIndex(String indexName) {
@@ -428,7 +437,7 @@ public class ProfileServiceImpl implements ProfileService, SynchronousBundleList
private void initializePurge() {
logger.info("Purge: Initializing");
- if (purgeProfileInactiveTime > 0 || purgeProfileExistTime > 0 || purgeSessionsAndEventsTime > 0 || purgeSessionExistTime > 0 || purgeEventExistTime > 0) {
+ if (purgeProfileInactiveTime > 0 || purgeProfileExistTime > 0 || purgeSessionExistTime > 0 || purgeEventExistTime > 0) {
if (purgeProfileInactiveTime > 0) {
logger.info("Purge: Profile with no visits since more than {} days, will be purged", purgeProfileInactiveTime);
}
diff --git a/services/src/main/resources/org.apache.unomi.services.cfg b/services/src/main/resources/org.apache.unomi.services.cfg
index fbd9dff7d..9cab89153 100644
--- a/services/src/main/resources/org.apache.unomi.services.cfg
+++ b/services/src/main/resources/org.apache.unomi.services.cfg
@@ -27,7 +27,9 @@ profile.purge.existTime=${org.apache.unomi.profile.purge.existTime:--1}
# Refresh Elasticsearch after saving a profile
profile.forceRefreshOnSave=${org.apache.unomi.profile.forceRefreshOnSave:-false}
-# Purge all monthly indexes (sessions/events) that have been created for a specific number of months
+# Deprecated: Use session.purge.existTime and event.purge.existTime instead.
+# If there is no value for session.purge.existTime or event.purge.existTime, the value
+# of this property will be use for the purge.
monthly.index.purge.existTime=${org.apache.unomi.monthly.index.purge.existTime:-12}
# Purge sessions that have been created for a specific number of days