You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2018/02/17 16:42:02 UTC
incubator-unomi git commit: UNOMI-163 Optimize
PropertyConditionEvaluator
Repository: incubator-unomi
Updated Branches:
refs/heads/master 69c4b8c71 -> 57a29f982
UNOMI-163 Optimize PropertyConditionEvaluator
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/57a29f98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/57a29f98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/57a29f98
Branch: refs/heads/master
Commit: 57a29f982c4dcb1e8dc7c1393a51fa6de23c302d
Parents: 69c4b8c
Author: Serge Huber <sh...@apache.org>
Authored: Sat Feb 17 17:41:58 2018 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Sat Feb 17 17:41:58 2018 +0100
----------------------------------------------------------------------
plugins/baseplugin/pom.xml | 15 ++
.../conditions/PropertyConditionEvaluator.java | 103 +++++++++-
.../PropertyConditionEvaluatorTest.java | 187 +++++++++++++++++++
3 files changed, 300 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57a29f98/plugins/baseplugin/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/pom.xml b/plugins/baseplugin/pom.xml
index ec0dfff..b0d2611 100644
--- a/plugins/baseplugin/pom.xml
+++ b/plugins/baseplugin/pom.xml
@@ -78,6 +78,21 @@
<artifactId>jackson-databind</artifactId>
<scope>provided</scope>
</dependency>
+
+ <!-- Unit tests -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.6.6</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57a29f98/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index 67178a6..0b97abf 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -24,8 +24,7 @@ import ognl.OgnlException;
import ognl.enhance.ExpressionAccessor;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.unomi.api.Event;
-import org.apache.unomi.api.Item;
+import org.apache.unomi.api.*;
import org.apache.unomi.api.conditions.Condition;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper;
import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator;
@@ -39,6 +38,7 @@ import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
import java.util.*;
+import java.util.concurrent.ConcurrentSkipListSet;
import java.util.function.LongSupplier;
import java.util.regex.Pattern;
@@ -50,6 +50,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
private static final Logger logger = LoggerFactory.getLogger(PropertyConditionEvaluator.class.getName());
private static final SimpleDateFormat yearMonthDayDateFormat = new SimpleDateFormat("yyyyMMdd");
+ public static final String NOT_OPTIMIZED_MARKER = "$$$###NOT_OPTIMIZED###$$$";
private Map<String, Map<String, ExpressionAccessor>> expressionCache = new HashMap<>(64);
@@ -244,11 +245,101 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
return false;
}
- private Object getPropertyValue(Item item, String expression) throws Exception {
+ protected Object getPropertyValue(Item item, String expression) throws Exception {
+ Object result = getHardcodedPropertyValue(item, expression);
+ if (!NOT_OPTIMIZED_MARKER.equals(result)) {
+ return result;
+ }
+ return getOGNLPropertyValue(item, expression);
+ }
+
+ protected Object getHardcodedPropertyValue(Item item, String expression) {
+ // the following are optimizations to avoid using the expressions that are slower. The main objective here is
+ // to avoid the most used expression that may also trigger calls to the Java Reflection API.
+ if (item instanceof Event) {
+ Event event = (Event) item;
+ if (expression.startsWith("properties.")) {
+ return getNestedPropertyValue(expression.substring("properties.".length()), event.getProperties());
+ }
+ if ("target.itemId".equals(expression)) {
+ return event.getTarget().getItemId();
+ }
+ if (expression.startsWith("target.properties.")) {
+ if (event.getTarget() instanceof CustomItem) {
+ CustomItem customItem = (CustomItem) event.getTarget();
+ String expressionPart = expression.substring("target.properties.".length());
+ return getNestedPropertyValue(expressionPart, customItem.getProperties());
+ }
+ }
+ if ("target.scope".equals(expression)) {
+ return event.getTarget().getScope();
+ }
+ if ("scope".equals(expression)) {
+ return event.getScope();
+ }
+ } else if (item instanceof Session) {
+ Session session = (Session) item;
+ if ("timeStamp".equals(expression)) {
+ return session.getTimeStamp();
+ }
+ if ("duration".equals(expression)) {
+ return session.getDuration();
+ }
+ if ("size".equals(expression)) {
+ return session.getSize();
+ }
+ if (expression.startsWith("properties.")) {
+ return getNestedPropertyValue(expression.substring("properties.".length()), session.getProperties());
+ }
+ if (expression.startsWith("systemProperties.")) {
+ return getNestedPropertyValue(expression.substring("systemProperties.".length()), session.getSystemProperties());
+ }
+ } else if (item instanceof Profile) {
+ Profile profile = (Profile) item;
+ if ("segments".equals(expression)) {
+ return profile.getSegments();
+ }
+ if ("consents".equals(expression)) {
+ return profile.getConsents();
+ }
+ if (expression.startsWith("scores.")) {
+ return profile.getScores().get(expression.substring("scores.".length()));
+ }
+ if (expression.startsWith("properties.")) {
+ return getNestedPropertyValue(expression.substring("properties.".length()), profile.getProperties());
+ }
+ if (expression.startsWith("systemProperties.")) {
+ return getNestedPropertyValue(expression.substring("systemProperties.".length()), profile.getSystemProperties());
+ }
+ } else if (item instanceof CustomItem) {
+ CustomItem customItem = (CustomItem) item;
+ if (expression.startsWith("properties.")) {
+ return getNestedPropertyValue(expression.substring("properties.".length()), customItem.getProperties());
+ }
+ }
+ return NOT_OPTIMIZED_MARKER;
+ }
+
+ protected Object getOGNLPropertyValue(Item item, String expression) throws Exception {
ExpressionAccessor accessor = getPropertyAccessor(item, expression);
return accessor != null ? accessor.get((OgnlContext) Ognl.createDefaultContext(null), item) : null;
}
+ private Object getNestedPropertyValue(String expressionPart, Map<String, Object> properties) {
+ int nextDotPos = expressionPart.indexOf(".");
+ if (nextDotPos > -1) {
+ String mapKey = expressionPart.substring(0, nextDotPos);
+ Object mapValue = properties.get(mapKey);
+ if (mapValue == null) {
+ return null;
+ }
+ String nextExpression = expressionPart.substring(nextDotPos+1);
+ return getNestedPropertyValue(nextExpression, (Map<String,Object>) mapValue);
+ } else {
+ return properties.get(expressionPart);
+ }
+ }
+
private ExpressionAccessor getPropertyAccessor(Item item, String expression) throws Exception {
ExpressionAccessor accessor = null;
String clazz = item.getClass().getName();
@@ -275,8 +366,10 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
} else {
logger.warn("Unable to compile expression for {} and {}", clazz, expression);
}
- time = System.nanoTime() - time;
- logger.info("Expression compilation for {} took {}", expression, time / 1000000L);
+ if (logger.isInfoEnabled()) {
+ time = System.nanoTime() - time;
+ logger.info("Expression compilation for item={} expression={} took {}", item.getClass().getName(), expression, time / 1000000L);
+ }
}
return accessor;
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57a29f98/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java b/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java
new file mode 100644
index 0000000..c3ba86c
--- /dev/null
+++ b/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.plugins.baseplugin.conditions;
+
+import org.apache.unomi.api.CustomItem;
+import org.apache.unomi.api.Event;
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.Session;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNull;
+import static org.apache.unomi.plugins.baseplugin.conditions.PropertyConditionEvaluator.NOT_OPTIMIZED_MARKER;
+
+public class PropertyConditionEvaluatorTest {
+
+ public static final String MOCK_ITEM_ID = "mockItemId";
+ public static final String DIGITALL_SCOPE = "digitall";
+ public static final String PAGE_PATH_VALUE = "/site/en/home/aboutus.html";
+ public static final String PAGE_URL_VALUE = "http://localhost:8080/site/en/home/aboutus.html";
+ public static final Date SESSION_LAST_EVENT_DATE = new Date();
+ private static PropertyConditionEvaluator propertyConditionEvaluator = new PropertyConditionEvaluator();
+ private static Event mockEvent = generateMockEvent();
+ private static Profile mockProfile = generateMockProfile();
+ private static Session mockSession = generateMockSession();
+
+ @Test
+ public void testHardcodedEvaluator() {
+ Event mockEvent = generateMockEvent();
+ assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.itemId"));
+ assertEquals("Target scope is not correct", DIGITALL_SCOPE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.scope"));
+ assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pagePath"));
+ assertEquals("Target page url value is not correct", PAGE_URL_VALUE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pageURL"));
+ assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "size"));
+
+ assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached"));
+ assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockProfile, "properties.email"));
+
+ // here let's make sure our reporting of non optimized expressions works.
+ assertEquals("Should have received the non-optimized marker string", NOT_OPTIMIZED_MARKER, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "lastEventDate") );
+
+ }
+
+ @Test
+ public void testOGNLEvaluator() throws Exception {
+ Event mockEvent = generateMockEvent();
+ assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.itemId"));
+ assertEquals("Target scope is not correct", DIGITALL_SCOPE,propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.scope"));
+ assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pagePath"));
+ assertEquals("Target page url value is not correct", PAGE_URL_VALUE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pageURL"));
+ assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "size"));
+ assertEquals("Should have received the proper last even date", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "lastEventDate") );
+
+ assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached"));
+ assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockProfile, "properties.email"));
+ }
+
+ @Test
+ public void testCompareOGNLvsHardcodedPerformance() throws InterruptedException {
+ int workerCount = 5000000;
+ ExecutorService executorService = Executors.newFixedThreadPool(3000);
+
+ runHardcodedTest(workerCount, executorService);
+ runOGNLTest(workerCount, executorService);
+ runHardcodedTest(workerCount, executorService);
+ runOGNLTest(workerCount, executorService);
+ runHardcodedTest(workerCount, executorService);
+ runOGNLTest(workerCount, executorService);
+ runHardcodedTest(workerCount, executorService);
+ runOGNLTest(workerCount, executorService);
+
+ }
+
+ @Test
+ public void testPropertyEvaluator() throws Exception {
+ Event mockEvent = generateMockEvent();
+ assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.itemId"));
+ assertEquals("Target scope is not correct", DIGITALL_SCOPE,propertyConditionEvaluator.getPropertyValue(mockEvent, "target.scope"));
+ assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.properties.pageInfo.pagePath"));
+
+ assertNull("Unexisting property should be null", propertyConditionEvaluator.getPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached"));
+ assertNull("Unexisting property should be null", propertyConditionEvaluator.getPropertyValue(mockProfile, "properties.email"));
+
+ assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getPropertyValue(mockSession, "size"));
+ assertEquals("Session last event date is not right", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getPropertyValue(mockSession, "lastEventDate"));
+ }
+
+ private void runHardcodedTest(int workerCount, ExecutorService executorService) throws InterruptedException {
+ List<Callable<Object>> todo = new ArrayList<Callable<Object>>(workerCount);
+ long startTime = System.currentTimeMillis();
+ for (int i = 0; i < workerCount; i++) {
+ todo.add(new HardcodedWorker());
+ }
+ List<Future<Object>> answers = executorService.invokeAll(todo);
+ long totalTime = System.currentTimeMillis() - startTime;
+ System.out.println("Hardcoded workers completed execution in " + totalTime + "ms");
+ }
+
+ private void runOGNLTest(int workerCount, ExecutorService executorService) throws InterruptedException {
+ List<Callable<Object>> todo = new ArrayList<Callable<Object>>(workerCount);
+ long startTime = System.currentTimeMillis();
+ for (int i = 0; i < workerCount; i++) {
+ todo.add(new OGNLWorker());
+ }
+ List<Future<Object>> answers = executorService.invokeAll(todo);
+ long totalTime = System.currentTimeMillis() - startTime;
+ System.out.println("OGNL workers completed execution in " + totalTime + "ms");
+ }
+
+ private static Event generateMockEvent() {
+ Event mockEvent = new Event();
+ CustomItem targetItem = new CustomItem();
+ targetItem.setItemId(MOCK_ITEM_ID);
+ targetItem.setScope(DIGITALL_SCOPE);
+ mockEvent.setTarget(targetItem);
+ Map<String,Object> pageInfoMap = new HashMap<>();
+ pageInfoMap.put("pagePath", PAGE_PATH_VALUE);
+ pageInfoMap.put("pageURL", PAGE_URL_VALUE);
+ targetItem.getProperties().put("pageInfo", pageInfoMap);
+ return mockEvent;
+ }
+
+ public static Profile generateMockProfile() {
+ Profile mockProfile = new Profile();
+ mockProfile.setItemId("mockProfileId");
+ mockProfile.getSegments().add("segment1");
+ mockProfile.getSegments().add("segment2");
+ mockProfile.getSegments().add("segment3");
+ return mockProfile;
+ }
+
+ public static Session generateMockSession() {
+ Session mockSession = new Session("mockSessionId", generateMockProfile(), new Date(), "digitall");
+ mockSession.setSize(10);
+ mockSession.setLastEventDate(SESSION_LAST_EVENT_DATE);
+ return mockSession;
+ }
+
+ class HardcodedWorker implements Callable<Object> {
+
+ @Override
+ public Object call() {
+ propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.itemId");
+ propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.scope");
+ propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pagePath");
+ propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pageURL");
+ return null;
+ }
+ }
+
+ class OGNLWorker implements Callable<Object> {
+
+ @Override
+ public Object call() {
+ try {
+ propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.itemId");
+ propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.scope");
+ propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pagePath");
+ propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pageURL");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+ }
+
+}