You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2018/02/17 16:42:02 UTC

incubator-unomi git commit: UNOMI-163 Optimize PropertyConditionEvaluator

Repository: incubator-unomi
Updated Branches:
  refs/heads/master 69c4b8c71 -> 57a29f982


UNOMI-163 Optimize PropertyConditionEvaluator


Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/57a29f98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/57a29f98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/57a29f98

Branch: refs/heads/master
Commit: 57a29f982c4dcb1e8dc7c1393a51fa6de23c302d
Parents: 69c4b8c
Author: Serge Huber <sh...@apache.org>
Authored: Sat Feb 17 17:41:58 2018 +0100
Committer: Serge Huber <sh...@apache.org>
Committed: Sat Feb 17 17:41:58 2018 +0100

----------------------------------------------------------------------
 plugins/baseplugin/pom.xml                      |  15 ++
 .../conditions/PropertyConditionEvaluator.java  | 103 +++++++++-
 .../PropertyConditionEvaluatorTest.java         | 187 +++++++++++++++++++
 3 files changed, 300 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57a29f98/plugins/baseplugin/pom.xml
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/pom.xml b/plugins/baseplugin/pom.xml
index ec0dfff..b0d2611 100644
--- a/plugins/baseplugin/pom.xml
+++ b/plugins/baseplugin/pom.xml
@@ -78,6 +78,21 @@
             <artifactId>jackson-databind</artifactId>
             <scope>provided</scope>
         </dependency>
+
+        <!-- Unit tests -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.11</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>1.6.6</version>
+            <scope>test</scope>
+        </dependency>
+
     </dependencies>
 
     <build>

http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57a29f98/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
index 67178a6..0b97abf 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java
@@ -24,8 +24,7 @@ import ognl.OgnlException;
 import ognl.enhance.ExpressionAccessor;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.unomi.api.Event;
-import org.apache.unomi.api.Item;
+import org.apache.unomi.api.*;
 import org.apache.unomi.api.conditions.Condition;
 import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper;
 import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator;
@@ -39,6 +38,7 @@ import org.slf4j.LoggerFactory;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.function.LongSupplier;
 import java.util.regex.Pattern;
 
@@ -50,6 +50,7 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
     private static final Logger logger = LoggerFactory.getLogger(PropertyConditionEvaluator.class.getName());
 
     private static final SimpleDateFormat yearMonthDayDateFormat = new SimpleDateFormat("yyyyMMdd");
+    public static final String NOT_OPTIMIZED_MARKER = "$$$###NOT_OPTIMIZED###$$$";
 
     private Map<String, Map<String, ExpressionAccessor>> expressionCache = new HashMap<>(64);
 
@@ -244,11 +245,101 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
         return false;
     }
 
-    private Object getPropertyValue(Item item, String expression) throws Exception {
+    protected Object getPropertyValue(Item item, String expression) throws Exception {
+        Object result = getHardcodedPropertyValue(item, expression);
+        if (!NOT_OPTIMIZED_MARKER.equals(result)) {
+            return result;
+        }
+        return getOGNLPropertyValue(item, expression);
+    }
+
+    protected Object getHardcodedPropertyValue(Item item, String expression) {
+        // the following are optimizations to avoid using the expressions that are slower. The main objective here is
+        // to avoid the most used expression that may also trigger calls to the Java Reflection API.
+        if (item instanceof Event) {
+            Event event = (Event) item;
+            if (expression.startsWith("properties.")) {
+                return getNestedPropertyValue(expression.substring("properties.".length()), event.getProperties());
+            }
+            if ("target.itemId".equals(expression)) {
+                return event.getTarget().getItemId();
+            }
+            if (expression.startsWith("target.properties.")) {
+                if (event.getTarget() instanceof CustomItem) {
+                    CustomItem customItem = (CustomItem) event.getTarget();
+                    String expressionPart = expression.substring("target.properties.".length());
+                    return getNestedPropertyValue(expressionPart, customItem.getProperties());
+                }
+            }
+            if ("target.scope".equals(expression)) {
+                return event.getTarget().getScope();
+            }
+            if ("scope".equals(expression)) {
+                return event.getScope();
+            }
+        } else if (item instanceof Session) {
+            Session session = (Session) item;
+            if ("timeStamp".equals(expression)) {
+                return session.getTimeStamp();
+            }
+            if ("duration".equals(expression)) {
+                return session.getDuration();
+            }
+            if ("size".equals(expression)) {
+                return session.getSize();
+            }
+            if (expression.startsWith("properties.")) {
+                return getNestedPropertyValue(expression.substring("properties.".length()), session.getProperties());
+            }
+            if (expression.startsWith("systemProperties.")) {
+                return getNestedPropertyValue(expression.substring("systemProperties.".length()), session.getSystemProperties());
+            }
+        } else if (item instanceof Profile) {
+            Profile profile = (Profile) item;
+            if ("segments".equals(expression)) {
+                return profile.getSegments();
+            }
+            if ("consents".equals(expression)) {
+                return profile.getConsents();
+            }
+            if (expression.startsWith("scores.")) {
+                return profile.getScores().get(expression.substring("scores.".length()));
+            }
+            if (expression.startsWith("properties.")) {
+                return getNestedPropertyValue(expression.substring("properties.".length()), profile.getProperties());
+            }
+            if (expression.startsWith("systemProperties.")) {
+                return getNestedPropertyValue(expression.substring("systemProperties.".length()), profile.getSystemProperties());
+            }
+        } else if (item instanceof CustomItem) {
+            CustomItem customItem = (CustomItem) item;
+            if (expression.startsWith("properties.")) {
+                return getNestedPropertyValue(expression.substring("properties.".length()), customItem.getProperties());
+            }
+        }
+        return NOT_OPTIMIZED_MARKER;
+    }
+
+    protected Object getOGNLPropertyValue(Item item, String expression) throws Exception {
         ExpressionAccessor accessor = getPropertyAccessor(item, expression);
         return accessor != null ? accessor.get((OgnlContext) Ognl.createDefaultContext(null), item) : null;
     }
 
+    private Object getNestedPropertyValue(String expressionPart, Map<String, Object> properties) {
+        int nextDotPos = expressionPart.indexOf(".");
+        if (nextDotPos > -1) {
+            String mapKey = expressionPart.substring(0, nextDotPos);
+            Object mapValue = properties.get(mapKey);
+            if (mapValue == null) {
+                return null;
+            }
+            String nextExpression = expressionPart.substring(nextDotPos+1);
+            return getNestedPropertyValue(nextExpression, (Map<String,Object>) mapValue);
+        } else {
+            return properties.get(expressionPart);
+        }
+    }
+
     private ExpressionAccessor getPropertyAccessor(Item item, String expression) throws Exception {
         ExpressionAccessor accessor = null;
         String clazz = item.getClass().getName();
@@ -275,8 +366,10 @@ public class PropertyConditionEvaluator implements ConditionEvaluator {
             } else {
                 logger.warn("Unable to compile expression for {} and {}", clazz, expression);
             }
-            time = System.nanoTime() - time;
-            logger.info("Expression compilation for {} took {}", expression, time / 1000000L);
+            if (logger.isInfoEnabled()) {
+                time = System.nanoTime() - time;
+                logger.info("Expression compilation for item={} expression={} took {}", item.getClass().getName(), expression, time / 1000000L);
+            }
         }
 
         return accessor;

http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/57a29f98/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java b/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java
new file mode 100644
index 0000000..c3ba86c
--- /dev/null
+++ b/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.plugins.baseplugin.conditions;
+
+import org.apache.unomi.api.CustomItem;
+import org.apache.unomi.api.Event;
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.Session;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNull;
+import static org.apache.unomi.plugins.baseplugin.conditions.PropertyConditionEvaluator.NOT_OPTIMIZED_MARKER;
+
+public class PropertyConditionEvaluatorTest {
+
+    public static final String MOCK_ITEM_ID = "mockItemId";
+    public static final String DIGITALL_SCOPE = "digitall";
+    public static final String PAGE_PATH_VALUE = "/site/en/home/aboutus.html";
+    public static final String PAGE_URL_VALUE = "http://localhost:8080/site/en/home/aboutus.html";
+    public static final Date SESSION_LAST_EVENT_DATE = new Date();
+    private static PropertyConditionEvaluator propertyConditionEvaluator = new PropertyConditionEvaluator();
+    private static Event mockEvent = generateMockEvent();
+    private static Profile mockProfile = generateMockProfile();
+    private static Session mockSession = generateMockSession();
+
+    @Test
+    public void testHardcodedEvaluator() {
+        Event mockEvent = generateMockEvent();
+        assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.itemId"));
+        assertEquals("Target scope is not correct", DIGITALL_SCOPE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.scope"));
+        assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pagePath"));
+        assertEquals("Target page url value is not correct", PAGE_URL_VALUE,                 propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pageURL"));
+        assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "size"));
+
+        assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached"));
+        assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockProfile, "properties.email"));
+
+        // here let's make sure our reporting of non optimized expressions works.
+        assertEquals("Should have received the non-optimized marker string", NOT_OPTIMIZED_MARKER, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "lastEventDate") );
+
+    }
+
+    @Test
+    public void testOGNLEvaluator() throws Exception {
+        Event mockEvent = generateMockEvent();
+        assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.itemId"));
+        assertEquals("Target scope is not correct", DIGITALL_SCOPE,propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.scope"));
+        assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pagePath"));
+        assertEquals("Target page url value is not correct", PAGE_URL_VALUE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pageURL"));
+        assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "size"));
+        assertEquals("Should have received the proper last even date", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "lastEventDate") );
+
+        assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached"));
+        assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockProfile, "properties.email"));
+    }
+
+    @Test
+    public void testCompareOGNLvsHardcodedPerformance() throws InterruptedException {
+        int workerCount = 5000000;
+        ExecutorService executorService = Executors.newFixedThreadPool(3000);
+
+        runHardcodedTest(workerCount, executorService);
+        runOGNLTest(workerCount, executorService);
+        runHardcodedTest(workerCount, executorService);
+        runOGNLTest(workerCount, executorService);
+        runHardcodedTest(workerCount, executorService);
+        runOGNLTest(workerCount, executorService);
+        runHardcodedTest(workerCount, executorService);
+        runOGNLTest(workerCount, executorService);
+
+    }
+
+    @Test
+    public void testPropertyEvaluator() throws Exception {
+        Event mockEvent = generateMockEvent();
+        assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.itemId"));
+        assertEquals("Target scope is not correct", DIGITALL_SCOPE,propertyConditionEvaluator.getPropertyValue(mockEvent, "target.scope"));
+        assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.properties.pageInfo.pagePath"));
+
+        assertNull("Unexisting property should be null", propertyConditionEvaluator.getPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached"));
+        assertNull("Unexisting property should be null", propertyConditionEvaluator.getPropertyValue(mockProfile, "properties.email"));
+
+        assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getPropertyValue(mockSession, "size"));
+        assertEquals("Session last event date is not right", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getPropertyValue(mockSession, "lastEventDate"));
+    }
+
+    private void runHardcodedTest(int workerCount, ExecutorService executorService) throws InterruptedException {
+        List<Callable<Object>> todo = new ArrayList<Callable<Object>>(workerCount);
+        long startTime = System.currentTimeMillis();
+        for (int i = 0; i < workerCount; i++) {
+            todo.add(new HardcodedWorker());
+        }
+        List<Future<Object>> answers = executorService.invokeAll(todo);
+        long totalTime = System.currentTimeMillis() - startTime;
+        System.out.println("Hardcoded workers completed execution in " + totalTime + "ms");
+    }
+
+    private void runOGNLTest(int workerCount, ExecutorService executorService) throws InterruptedException {
+        List<Callable<Object>> todo = new ArrayList<Callable<Object>>(workerCount);
+        long startTime = System.currentTimeMillis();
+        for (int i = 0; i < workerCount; i++) {
+            todo.add(new OGNLWorker());
+        }
+        List<Future<Object>> answers = executorService.invokeAll(todo);
+        long totalTime = System.currentTimeMillis() - startTime;
+        System.out.println("OGNL workers completed execution in " + totalTime + "ms");
+    }
+
+    private static Event generateMockEvent() {
+        Event mockEvent = new Event();
+        CustomItem targetItem = new CustomItem();
+        targetItem.setItemId(MOCK_ITEM_ID);
+        targetItem.setScope(DIGITALL_SCOPE);
+        mockEvent.setTarget(targetItem);
+        Map<String,Object> pageInfoMap = new HashMap<>();
+        pageInfoMap.put("pagePath", PAGE_PATH_VALUE);
+        pageInfoMap.put("pageURL", PAGE_URL_VALUE);
+        targetItem.getProperties().put("pageInfo", pageInfoMap);
+        return mockEvent;
+    }
+
+    public static Profile generateMockProfile() {
+        Profile mockProfile = new Profile();
+        mockProfile.setItemId("mockProfileId");
+        mockProfile.getSegments().add("segment1");
+        mockProfile.getSegments().add("segment2");
+        mockProfile.getSegments().add("segment3");
+        return mockProfile;
+    }
+
+    public static Session generateMockSession() {
+        Session mockSession = new Session("mockSessionId", generateMockProfile(), new Date(), "digitall");
+        mockSession.setSize(10);
+        mockSession.setLastEventDate(SESSION_LAST_EVENT_DATE);
+        return mockSession;
+    }
+
+    class HardcodedWorker implements Callable<Object> {
+
+        @Override
+        public Object call() {
+            propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.itemId");
+            propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.scope");
+            propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pagePath");
+            propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pageURL");
+            return null;
+        }
+    }
+
+    class OGNLWorker implements Callable<Object> {
+
+        @Override
+        public Object call() {
+            try {
+                propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.itemId");
+                propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.scope");
+                propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pagePath");
+                propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pageURL");
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+            return null;
+        }
+    }
+
+}