You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by at...@apache.org on 2017/12/09 21:26:13 UTC

[2/2] commons-scxml git commit: SCXML-255 Use jexl for SCXML specification IRP test verification

SCXML-255 Use jexl for SCXML specification IRP test verification

Adapted the original confEcma.xsl in confJexl.xsl to produce Jexl datamodel based scxml test definitions.
Currently 98 Jexl tests pass, 70 failing (for ecmascript: 111 pass, 77 failing)


Project: http://git-wip-us.apache.org/repos/asf/commons-scxml/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-scxml/commit/321fd426
Tree: http://git-wip-us.apache.org/repos/asf/commons-scxml/tree/321fd426
Diff: http://git-wip-us.apache.org/repos/asf/commons-scxml/diff/321fd426

Branch: refs/heads/master
Commit: 321fd426406b6e60b6ea1bf9bf60857d6484f0f4
Parents: fdf7e16
Author: Ate Douma <at...@apache.org>
Authored: Sat Dec 9 22:26:04 2017 +0100
Committer: Ate Douma <at...@apache.org>
Committed: Sat Dec 9 22:26:04 2017 +0100

----------------------------------------------------------------------
 pom.xml                                         |   2 +
 src/changes/changes.xml                         |   4 +
 .../org/apache/commons/scxml2/w3c/W3CTests.java | 334 ++++++++-------
 .../org/apache/commons/scxml2/w3c/confJexl.xsl  |  45 +--
 .../org/apache/commons/scxml2/w3c/tests.xml     | 401 ++++++++++---------
 5 files changed, 386 insertions(+), 400 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/commons-scxml/blob/321fd426/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0d18773..d6d09ea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -282,6 +282,8 @@
           <excludes combine.children="append">
             <!--exclude stylesheet adapted from http://www.w3.org/Voice/2013/scxml-irp/confXPath.xsl -->
             <exclude>src/test/java/org/apache/commons/scxml2/w3c/confMinimal.xsl</exclude>
+            <!--exclude stylesheet adapted from http://www.w3.org/Voice/2013/scxml-irp/confEcma.xsl -->
+            <exclude>src/test/java/org/apache/commons/scxml2/w3c/confJexl.xsl</exclude>
           </excludes>
         </configuration>
       </plugin>

http://git-wip-us.apache.org/repos/asf/commons-scxml/blob/321fd426/src/changes/changes.xml
----------------------------------------------------------------------
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 5b9bf16..3b0e12c 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -35,6 +35,10 @@
     <release version="2.0" date="In Git master"
       description="Latest unreleased code">
 
+      <action dev="ate" type="update" issue="SCXML-255">
+        [12-09-2017] Use jexl for SCXML specification IRP test verification
+      </action>
+
       <action dev="ate" type="update" issue="SCXML-254">
         [12-09-2017] Update to current dependencies for groovy, commons-jexl, commons-io, jackson
       </action>

http://git-wip-us.apache.org/repos/asf/commons-scxml/blob/321fd426/src/test/java/org/apache/commons/scxml2/w3c/W3CTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/scxml2/w3c/W3CTests.java b/src/test/java/org/apache/commons/scxml2/w3c/W3CTests.java
index 627bd58..edf7c5c 100644
--- a/src/test/java/org/apache/commons/scxml2/w3c/W3CTests.java
+++ b/src/test/java/org/apache/commons/scxml2/w3c/W3CTests.java
@@ -23,8 +23,10 @@ import java.io.FileReader;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.Unmarshaller;
@@ -59,7 +61,7 @@ import org.apache.commons.scxml2.model.SCXML;
  * To execute one or multiple IRP tests the commandline parameter <b>run</b> must be specified.
  * </p>
  * <p>
- * Optional environment parameter <b>-Ddatamodel=&lt;minimal|ecma&gt;</b> can be specified to limit the
+ * Optional environment parameter <b>-Ddatamodel=&lt;minimal|ecma|jexl&gt;</b> can be specified to limit the
  * execution of the tests for and using only the specified datamodel language.
  * </p>
  * <p>
@@ -83,11 +85,51 @@ public class W3CTests {
 
     private static final String TESTS_SRC_DIR = "src/w3c/scxml-irp/";
     private static final String TXML_TESTS_DIR = TESTS_SRC_DIR + "txml/";
-    private static final String MINIMAL_TESTS_DIR = TESTS_SRC_DIR + "minimal/";
-    private static final String ECMA_TESTS_DIR = TESTS_SRC_DIR + "ecma/";
     private static final String PACKAGE_PATH = "/"+W3CTests.class.getPackage().getName().replace('.','/');
     private static final String TESTS_FILENAME = PACKAGE_PATH + "/tests.xml";
     private static final String SCXML_IRP_MINIMAL_XSL_FILENAME = PACKAGE_PATH + "/confMinimal.xsl";
+    private static final String SCXML_IRP_JEXL_XSL_FILENAME = PACKAGE_PATH + "/confJexl.xsl";
+
+    /**
+     * Datamodel enum representing the datamodel types used and tested with the W3C IRP tests.
+     */
+    protected enum Datamodel {
+
+        MINIMAL("minimal", "minimal"),
+        ECMA("ecma",       "ecma   "),
+        JEXL("jexl",       "jexl   ");
+
+        private final String value;
+        private final String label;
+        private final String testDir;
+
+        Datamodel(final String value, final String label) {
+            this.value = value;
+            this.label = label;
+            this.testDir = TESTS_SRC_DIR + value + "/";
+        }
+
+        public String value() {
+            return value;
+        }
+
+        public String label() {
+            return label;
+        }
+
+        public String testDir() {
+            return testDir;
+        }
+
+        public static Datamodel fromValue(final String value) {
+            for (Datamodel datamodel : Datamodel.values()) {
+                if (datamodel.value().equals(value)) {
+                    return datamodel;
+                }
+            }
+            return null;
+        }
+    }
 
     /**
      * Tests model class used for loading the <b>tests.xml</b> configuration file
@@ -105,16 +147,16 @@ public class W3CTests {
             private Boolean mandatory;
             @XmlAttribute(required=true)
             private Boolean manual;
-            @XmlAttribute(required=true)
-            private boolean enabled;
             @XmlAttribute
-            private String finalId;
+            private String finalState;
             @XmlAttribute
             private Boolean implemented;
             @XmlAttribute(name="minimal")
-            String minimalStatus;
+            private Boolean minimalStatus;
             @XmlAttribute(name="ecma")
-            String ecmaStatus;
+            private Boolean ecmaStatus;
+            @XmlAttribute(name="jexl")
+            private boolean jexlStatus;
             @XmlValue
             private String comment;
 
@@ -130,26 +172,24 @@ public class W3CTests {
                 return manual == null || manual;
             }
 
-            public boolean isEnabled() {
-                return enabled;
-            }
-
             public String getFinalState() {
-                return finalId;
+                return finalState;
             }
 
             public boolean isImplemented() {
                 return implemented == null || implemented;
             }
 
-            public String getMinimalStatus() {
-                return minimalStatus;
-            }
-
-            public String getEcmaStatus() {
-                return ecmaStatus;
+            public Boolean getStatus(final Datamodel dm) {
+                switch (dm) {
+                    case ECMA:
+                        return ecmaStatus;
+                    case JEXL:
+                        return jexlStatus;
+                    default:
+                        return minimalStatus;
+                }
             }
-
             public String getComment() {
                 return comment;
             }
@@ -178,31 +218,14 @@ public class W3CTests {
     }
 
     /**
-     * Datamodel enum representing the minimal and ecma datamodel types used and tested by the W3C IRP tests.
+     * Loads the tests.xml configuration file into a Tests class configuration model instance.
+     * @return a Tests instance for the tests.xml configuration file.
+     * @throws Exception
      */
-    protected enum Datamodel {
-
-        MINIMAL("minimal"),
-        ECMA("ecma");
-
-        private final String value;
-
-        private Datamodel(final String value) {
-            this.value = value;
-        }
-
-        public String value() {
-            return value;
-        }
-
-        public static Datamodel fromValue(final String value) {
-            for (Datamodel datamodel : Datamodel.values()) {
-                if (datamodel.value().equals(value)) {
-                    return datamodel;
-                }
-            }
-            return null;
-        }
+    protected Tests loadTests() throws Exception {
+        final JAXBContext jaxbContext = JAXBContext.newInstance(Tests.class);
+        final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+        return (Tests)jaxbUnmarshaller.unmarshal(getClass().getResource(TESTS_FILENAME));
     }
 
     /**
@@ -342,17 +365,36 @@ public class W3CTests {
     }
 
     /**
+     * Unmarshall and return the W3C IRP tests manifest.xml
+     * @return an Assertions instance reprenting the W3C IRP tests manifest.xml
+     * @throws Exception
+     */
+    protected Assertions loadAssertions() throws Exception {
+        final JAXBContext jaxbContext = JAXBContext.newInstance(Assertions.class);
+        final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
+        return (Assertions)jaxbUnmarshaller.unmarshal(new File(TESTS_SRC_DIR, SCXML_IRP_MANIFEST_URI));
+    }
+
+    /**
      * Simple TestResult data struct for tracking test results
      */
     protected static class TestResults {
-        int testsSkipped;
-        int testsPassed;
-        int testsFailed;
-        int minimalPassed;
-        int minimalFailed;
-        int ecmaPassed;
-        int ecmaFailed;
-        ArrayList<String> failedTests = new ArrayList<>();
+        Map<Datamodel, Integer> passed = new HashMap<>();
+        Map<Datamodel, Integer> failed = new HashMap<>();
+        Map<Datamodel, Integer> skipped = new HashMap<>();
+        ArrayList<String> changedStatusTests = new ArrayList<>();
+
+        public int passed(final Datamodel dm) {
+            return passed.get(dm) != null ? passed.get(dm) : 0;
+        }
+
+        public int failed(final Datamodel dm) {
+            return failed.get(dm) != null ? failed.get(dm) : 0;
+        }
+
+        public int skipped(final Datamodel dm) {
+            return skipped.get(dm) != null ? skipped.get(dm) : 0;
+        }
     }
 
     /**
@@ -387,9 +429,11 @@ public class W3CTests {
         System.out.println("Usage: W3CTests <get|run>\n" +
                 "  get  - downloads the W3C IRP tests\n" +
                 "  make - make previously downloaded  W3C IRP tests by transforming the .txml templates\n" +
-                "  run  - runs test(s), optionally only for a specific datamodel (default: all)\n\n" +
-                "To run a single test, specify -Dtest=<testId>, otherwise all enabled tests will be run.\n" +
-                "To only run test(s) for a specific datamodel, specify -Ddatamodel=<minimal|ecma>.\n");
+                "  run  - runs test(s), optionally only for a specific datamodel (default: all)\n" +
+                "         To run a single test, specify -Dtest=<testId>, otherwise all tests will be run.\n" +
+                "         To only run test(s) for a specific datamodel, specify -Ddatamodel=<minimal|ecma|jexl>.\n" +
+                "         By default only enabled tests (for the specified datamodel, or all) are run,\n" +
+                "         specify -Denabled=false to only run disabled tests.\n");
     }
 
     /**
@@ -403,8 +447,9 @@ public class W3CTests {
             FileUtils.cleanDirectory(testsSrcDir);
         }
         new File(TXML_TESTS_DIR).mkdirs();
-        new File(MINIMAL_TESTS_DIR).mkdirs();
-        new File(ECMA_TESTS_DIR).mkdirs();
+        for (final Datamodel dm : Datamodel.values()) {
+            new File(dm.testDir()).mkdirs();
+        }
         System.out.println("Downloading IRP manifest: " + SCXML_IRP_BASE_URL + SCXML_IRP_MANIFEST_URI);
         FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + SCXML_IRP_MANIFEST_URI), new File(testsSrcDir, SCXML_IRP_MANIFEST_URI));
         System.out.println("Downloading ecma stylesheet: " + SCXML_IRP_BASE_URL + SCXML_IRP_ECMA_XSL_URI);
@@ -432,51 +477,44 @@ public class W3CTests {
 
         TransformerFactory factory = TransformerFactory.newInstance("net.sf.saxon.TransformerFactoryImpl",null);
         factory.setFeature("http://saxon.sf.net/feature/suppressXsltNamespaceCheck", true);
-        Transformer ecmaTransformer = factory.newTransformer(new StreamSource(new FileInputStream(new File(testsSrcDir, SCXML_IRP_ECMA_XSL_URI))));
-        Transformer minimalTransformer = factory.newTransformer(new StreamSource(getClass().getResourceAsStream(SCXML_IRP_MINIMAL_XSL_FILENAME)));
+        final Map<Datamodel, Transformer> transformers = new HashMap<>();
+        transformers.put(Datamodel.ECMA, factory.newTransformer(new StreamSource(new FileInputStream(new File(testsSrcDir, SCXML_IRP_ECMA_XSL_URI)))));
+        transformers.put(Datamodel.MINIMAL, factory.newTransformer(new StreamSource(getClass().getResourceAsStream(SCXML_IRP_MINIMAL_XSL_FILENAME))));
+        transformers.put(Datamodel.JEXL, factory.newTransformer(new StreamSource(getClass().getResourceAsStream(SCXML_IRP_JEXL_XSL_FILENAME))));
         Assertions assertions = loadAssertions();
         for (Assertions.Assertion entry : assertions.getAssertions().values()) {
             for (Assertions.TestCase test : entry.getTestCases()) {
                 for (Assertions.Resource resource : test.getResources()) {
-                    processResource(entry.getSpecId(), resource, minimalTransformer, ecmaTransformer);
+                    processResource(entry.getSpecId(), resource, transformers);
                 }
             }
         }
     }
 
     /**
-     * Unmarshall and return the W3C IRP tests manifest.xml
-     * @return an Assertions instance reprenting the W3C IRP tests manifest.xml
-     * @throws Exception
-     */
-    protected Assertions loadAssertions() throws Exception {
-        final JAXBContext jaxbContext = JAXBContext.newInstance(Assertions.class);
-        final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
-        return (Assertions)jaxbUnmarshaller.unmarshal(new File(TESTS_SRC_DIR, SCXML_IRP_MANIFEST_URI));
-    }
-
-    /**
      * Download and transform a W3C IRP test resource file
      * @param specid the SCXML 1.0 spec id (anchor) for the current assertion,
      *               which is used to determine if, how and where the resource should be transformed.
      * @param resource The test resource definition
-     * @param minimalTransformer transformer to produce an minimal datamodel SCXML document from the txml resource
-     * @param ecmaTransformer transformer to produce an ecmascript datamodel SCXML document from the txml resource
+     * @param transformers map of datamodel transformers to produce a datamodel specific SCXML document from the txml resource
      * @throws Exception
      */
-    protected void processResource(final String specid, final Assertions.Resource resource,
-                                   final Transformer minimalTransformer, final Transformer ecmaTransformer)
+    protected void processResource(final String specid, final Assertions.Resource resource, final Map<Datamodel, Transformer> transformers)
             throws Exception {
         System.out.println("processing IRP test file " + resource.getFilename());
         FileUtils.copyURLToFile(new URL(SCXML_IRP_BASE_URL + resource.getUri()), new File(TXML_TESTS_DIR + resource.getFilename()));
         if (specid.equals("#minimal-profile")) {
-            transformResource(resource, minimalTransformer, MINIMAL_TESTS_DIR);
+            transformResource(resource, transformers.get(Datamodel.MINIMAL), Datamodel.MINIMAL.testDir());
         }
         else if (specid.equals("#ecma-profile")) {
-            transformResource(resource, ecmaTransformer, ECMA_TESTS_DIR);
+            transformResource(resource, transformers.get(Datamodel.ECMA), Datamodel.ECMA.testDir());
         }
         else {
-            transformResource(resource, ecmaTransformer, ECMA_TESTS_DIR);
+            for (Datamodel dm : transformers.keySet()) {
+                if (dm != Datamodel.MINIMAL) {
+                    transformResource(resource, transformers.get(dm), dm.testDir());
+                }
+            }
         }
     }
 
@@ -516,10 +554,11 @@ public class W3CTests {
         final Assertions assertions = loadAssertions();
         final Tests tests = loadTests();
         final TestResults results = new TestResults();
+        final boolean enabled = Boolean.parseBoolean(System.getProperty("enabled", "true"));
         if (testId != null) {
             final Assertions.Assertion assertion = assertions.getAssertions().get(testId);
             if (assertion != null) {
-                runTest(assertion, tests, datamodel, true, results);
+                runAssert(assertion, tests, datamodel, enabled, true, results);
             }
             else {
                 throw new IllegalArgumentException("Unknown test with id: "+testId);
@@ -527,28 +566,26 @@ public class W3CTests {
         }
         else {
             for (Assertions.Assertion entry : assertions.getAssertions().values()) {
-                runTest(entry, tests, datamodel, false, results);
+                runAssert(entry, tests, datamodel, enabled, false, results);
             }
         }
         System.out.println(
                 "\nTest results running " +
-                (testId == null ? "all enabled tests" : "test "+testId) +
-                (datamodel != null ? " for the "+datamodel.value+" datamodel" : "") +
-                ":\n" +
-                "  number of tests    : "+(results.testsSkipped+results.testsPassed+results.testsFailed) +
-                   " ("+results.testsPassed+" passed,  "+results.testsFailed +" failed,  "+results.testsSkipped+" skipped)");
-        if (results.minimalPassed+results.minimalFailed > 0) {
-            System.out.println(
-                    "    mimimal datamodel: "+results.minimalPassed+" passed,  "+results.minimalFailed+" failed");
-        }
-        if (results.ecmaPassed+results.ecmaFailed > 0) {
-            System.out.println(
-                    "    ecma    datamodel: "+results.ecmaPassed+" passed,  "+results.ecmaFailed+" failed");
+                (testId == null ? "all tests" : "test "+testId) +
+                (datamodel != null ? " for the "+datamodel.value+" datamodel" : "") + (enabled ? " enabled" : " disabled"));
+        for (final Datamodel dm : Datamodel.values()) {
+            if (datamodel == null || datamodel == dm) {
+                System.out.println(
+                        "    "+dm.label()+" datamodel: "+results.passed(dm)+" passed,  " +
+                                results.failed(dm)+" failed, " +
+                                results.skipped(dm)+" skipped ("+
+                                (results.passed(dm)+results.failed(dm)+results.skipped(dm))+" total)");
+            }
         }
         System.out.print("\n");
-        if (!results.failedTests.isEmpty()) {
-            System.out.println("  failed tests: ");
-            for (String filename : results.failedTests) {
+        if (testId == null && !results.changedStatusTests.isEmpty()) {
+            System.out.println("  "+(enabled? "failed" : "passed")+" tests: ");
+            for (String filename : results.changedStatusTests) {
                 System.out.println("    "+filename);
             }
             System.out.print("\n");
@@ -556,106 +593,49 @@ public class W3CTests {
     }
 
     /**
-     * Loads the tests.xml configuration file into a Tests class configuration model instance.
-     * @return a Tests instance for the tests.xml configuration file.
-     * @throws Exception
-     */
-    protected Tests loadTests() throws Exception {
-        final JAXBContext jaxbContext = JAXBContext.newInstance(Tests.class);
-        final Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
-        return (Tests)jaxbUnmarshaller.unmarshal(getClass().getResource(TESTS_FILENAME));
-    }
-
-    /**
-     * Run a single W3C IRP test (assert)
+     * Run a single W3C IRP assert test
      * @param assertion The W3C IRP assert, defining one or more {@link Assertions.TestCase}s
      * @param tests the tests configurations
      * @param datamodel the datamodel to limit and restrict the execution of the test
+     * @param status true to run the test with status true for the (or any) datamodel, false to do so for status false
      * @param singleTest if true a single test id was specified which will be executed even if disabled in the configuration.
      * @throws Exception
      */
-    protected void runTest(final Assertions.Assertion assertion, final Tests tests, final Datamodel datamodel,
-                           final boolean singleTest, TestResults results) throws Exception {
+    protected void runAssert(final Assertions.Assertion assertion, final Tests tests, final Datamodel datamodel,
+                             final boolean status, final boolean singleTest, TestResults results) throws Exception {
         final Tests.Test test = tests.getTests().get(assertion.getId());
         if (test == null) {
             throw new IllegalStateException("No test configuration found for W3C IRP test with id: "+assertion.getId());
         }
-        boolean skipped = true;
-        boolean passed = true;
-        if (singleTest || test.isEnabled()) {
-            if (datamodel != Datamodel.MINIMAL || datamodel.equals(assertion.getDatamodel())) {
-                if (datamodel == null || assertion.getDatamodel() == null || datamodel.equals(assertion.getDatamodel())) {
-                    final Datamodel effectiveDM = datamodel != null ? datamodel : assertion.getDatamodel();
-                    for (Assertions.TestCase testCase : assertion.getTestCases()) {
-                        if (effectiveDM != null) {
-                            switch (effectiveDM) {
-                                case MINIMAL:
-                                    skipped = false;
-                                    if (runTests(assertion, testCase, test, MINIMAL_TESTS_DIR, results.failedTests)) {
-                                        results.minimalPassed++;
-                                    }
-                                    else {
-                                        passed = false;
-                                        results.minimalFailed++;
-                                    }
-                                    break;
-                                case ECMA:
+        if (test.isImplemented()) {
+            for (Assertions.TestCase testCase : assertion.getTestCases()) {
+                for (final Datamodel dm : Datamodel.values()) {
+                    if (assertion.getDatamodel() == null && dm != Datamodel.MINIMAL || dm == assertion.getDatamodel()) {
+                        boolean skipped = true;
+                        if (datamodel == null || datamodel == dm) {
+                            if (singleTest || test.getStatus(dm) == null || status == test.getStatus(dm)) {
+                                for (Assertions.Resource scxmlResource : testCase.getScxmlResources()) {
+                                    File scxmlFile = new File(dm.testDir(), scxmlResource.getName()+".scxml");
                                     skipped = false;
-                                    if (runTests(assertion, testCase, test, ECMA_TESTS_DIR, results.failedTests)) {
-                                        results.ecmaPassed++;
+                                    boolean success = runTest(testCase, test, scxmlFile);
+                                    if (!success) {
+                                        results.failed.put(dm, results.failed(dm)+1);
+                                    } else {
+                                        results.passed.put(dm, results.passed(dm)+1);
                                     }
-                                    else {
-                                        passed = false;
-                                        results.ecmaFailed++;
+                                    if (success != status) {
+                                        results.changedStatusTests.add(scxmlFile.getParentFile().getName()+"/"+scxmlFile.getName());
                                     }
-                                    break;
+                                }
                             }
                         }
-                        else {
-                            skipped = false;
-                            if (runTests(assertion, testCase, test, ECMA_TESTS_DIR, results.failedTests)) {
-                                results.ecmaPassed++;
-                            }
-                            else {
-                                passed = false;
-                                results.ecmaFailed++;
-                            }
+                        if (skipped) {
+                            results.skipped.put(dm, results.skipped(dm)+1);
                         }
                     }
                 }
             }
         }
-        if (skipped) {
-            results.testsSkipped++;
-        }
-        else if (passed) {
-            results.testsPassed++;
-        }
-        else {
-            results.testsFailed++;
-        }
-    }
-
-    /**
-     * Execute all W3C IRP SCXML tests for a specific {@link Assertions.TestCase}
-     * @param assertion the W3C IRP test assert definition
-     * @param testCase the W3C IRP test definition
-     * @param test the test configuration
-     * @param scxmlDir the datamodel specific directory path containing the SCXML document(s)
-     * @throws Exception
-     */
-    protected boolean runTests(final Assertions.Assertion assertion, final Assertions.TestCase testCase,
-                               final Tests.Test test, final String scxmlDir, ArrayList<String> failedTests)
-            throws Exception {
-        boolean passed = true;
-        for (Assertions.Resource scxmlResource : testCase.getScxmlResources()) {
-            File scxmlFile = new File(scxmlDir, scxmlResource.getName()+".scxml");
-            if (!runTest(testCase, test, scxmlFile)) {
-                passed = false;
-                failedTests.add(scxmlFile.getParentFile().getName()+"/"+scxmlFile.getName());
-            }
-        }
-        return passed;
     }
 
     /**
@@ -701,6 +681,10 @@ public class W3CTests {
             }
         }
         catch (Exception e) {
+            if (test.isManual() && e.getMessage() != null && e.getMessage().equals(test.getFinalState())) {
+                System.out.println("                PASS: "+e.getMessage());
+                return true;
+            }
             System.out.println("                FAIL: "+e.getMessage());
             return false;
         }

http://git-wip-us.apache.org/repos/asf/commons-scxml/blob/321fd426/src/test/java/org/apache/commons/scxml2/w3c/confJexl.xsl
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/scxml2/w3c/confJexl.xsl b/src/test/java/org/apache/commons/scxml2/w3c/confJexl.xsl
index fa76485..e401778 100644
--- a/src/test/java/org/apache/commons/scxml2/w3c/confJexl.xsl
+++ b/src/test/java/org/apache/commons/scxml2/w3c/confJexl.xsl
@@ -1,5 +1,6 @@
 <?xml version="1.0"?>
 <!-- Copyright 1998-2003 W3C (MIT, ERCIM, Keio), All Rights Reserved. See http://www.w3.org/Consortium/Legal/. -->
+<!-- Adapted from http://www.w3.org/Voice/2013/scxml-irp/confEcma.xsl -->
 <xsl:stylesheet
     xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
     xmlns:conf="http://www.w3.org/2005/scxml-conformance"
@@ -45,7 +46,7 @@
 
 <!-- datamodel -->
 <xsl:template match="//@conf:datamodel">
-	<xsl:attribute name="datamodel">ecmascript</xsl:attribute>
+	<xsl:attribute name="datamodel">jexl</xsl:attribute>
 </xsl:template>
 
 
@@ -68,7 +69,7 @@
 
 <!-- names an invalid location for <assign>, etc. -->
 <xsl:template match="//@conf:invalidLocation">
-	<xsl:attribute name="location">foo.bar.baz </xsl:attribute>
+	<xsl:attribute name="location">foo.bar.baz[0]</xsl:attribute>
 </xsl:template>
 
 <!-- uses system var as location for <assign>, etc. -->
@@ -325,7 +326,7 @@ events which cause the test to fail.  The default value provided here is pretty
 					<xsl:matching-substring>Var<xsl:value-of select="regex-group(1)"/>
 						<xsl:variable name="op"><xsl:value-of select="regex-group(2)"/></xsl:variable>
 						<xsl:choose>
-							<xsl:when test="$op='='">===</xsl:when>
+							<xsl:when test="$op='='">==</xsl:when>
 							<xsl:otherwise><xsl:value-of select="$op"/></xsl:otherwise>
 					 </xsl:choose>
 							<xsl:value-of select="regex-group(3)"/>
@@ -340,7 +341,7 @@ events which cause the test to fail.  The default value provided here is pretty
 		<xsl:attribute name="cond">
 		<xsl:analyze-string select="."
 			regex="([0-9]+)(\W+)([0-9]+)">
-					<xsl:matching-substring>Var<xsl:value-of select="regex-group(1)"/>===Var<xsl:value-of select="regex-group(3)"/>
+					<xsl:matching-substring>Var<xsl:value-of select="regex-group(1)"/>==Var<xsl:value-of select="regex-group(3)"/>
 					</xsl:matching-substring>
 		</xsl:analyze-string>
 	</xsl:attribute>
@@ -444,12 +445,12 @@ is the second argument -->
 </xsl:template>
 
 <xsl:template match="//@conf:emptyEventData">
-	<xsl:attribute name="cond">typeof _event.data === 'undefined'</xsl:attribute>
+	<xsl:attribute name="cond">empty(_event.data</xsl:attribute>
 </xsl:template>
 
 <!-- return true if the _name system var has the specified quoted value -->
 <xsl:template match="//@conf:nameVarVal">
-	<xsl:attribute name="cond">_name  === '<xsl:value-of select="."/>'</xsl:attribute>
+	<xsl:attribute name="cond">_name  == '<xsl:value-of select="."/>'</xsl:attribute>
 </xsl:template>
 
 <!-- return true if first var's value is a prefix of the second var's value.  Input has form "n m" where n and m are ints.-->
@@ -458,14 +459,8 @@ is the second argument -->
 		<xsl:analyze-string select="."
 			regex="(\w+)(\W)(\w+)">
 					<xsl:matching-substring>
-					<!-- the underscore.string.startswith function compressed into one line below: 
-							<xsl:text>(function(str, starts){
-							      if (starts === '') return true;
-							      if (str == null || starts == null) return false;
-							      str = String(str); starts = String(starts);
-							      return str.length >= starts.length &amp;&amp; str.slice(0, starts.length) === starts;
-							    })(</xsl:text>Var<xsl:value-of select="regex-group(3)"/>, Var<xsl:value-of select="regex-group(1)"/><xsl:text>)</xsl:text> -->
-<xsl:text>(function(str, starts){if (starts === '') return true;if (str == null || starts == null) return false;str = String(str); starts = String(starts);return str.length >= starts.length &amp;&amp; str.slice(0, starts.length) === starts;})(</xsl:text>Var<xsl:value-of select="regex-group(3)"/>, Var<xsl:value-of select="regex-group(1)"/><xsl:text>)</xsl:text>
+						<!-- input "2 1" generates: "!empty(Var1) and !empty(Var2) and Var1 =^ Var2" -->
+						<xsl:text>!empty(Var</xsl:text><xsl:value-of select="regex-group(3)"/><xsl:text>) and !empty(Var</xsl:text><xsl:value-of select="regex-group(1)"/><xsl:text>) and Var</xsl:text><xsl:value-of select="regex-group(3)"/><xsl:text> =^ Var</xsl:text><xsl:value-of select="regex-group(1)"/>
 					</xsl:matching-substring>
 		</xsl:analyze-string>
 	</xsl:attribute>
@@ -482,22 +477,22 @@ is the second argument -->
 
 <!-- true if id has a value -->
 <xsl:template match="//@conf:isBound">
-	<xsl:attribute name="cond">Var<xsl:value-of select="." /></xsl:attribute>
+	<xsl:attribute name="cond">!empty(Var<xsl:value-of select="." />)</xsl:attribute>
 </xsl:template>
 
 <!-- return true if specified var has been created but is not bound -->
 <xsl:template match="//@conf:unboundVar">
-	<xsl:attribute name="cond">typeof Var<xsl:value-of select="." /> === 'undefined' </xsl:attribute>
+	<xsl:attribute name="cond">empty(Var<xsl:value-of select="." />)</xsl:attribute>
 </xsl:template>
 
 <!-- true if system var has a value -->
 <xsl:template match="//@conf:systemVarIsBound">
-	<xsl:attribute name="cond"><xsl:value-of select="." /></xsl:attribute>
+	<xsl:attribute name="cond">!empty(<xsl:value-of select="." />)</xsl:attribute>
 </xsl:template>
 
 <!-- true if id does not have a value -->
 <xsl:template match="//@conf:noValue">
-	<xsl:attribute name="cond">!Var<xsl:value-of select="." /></xsl:attribute>
+	<xsl:attribute name="cond">empty(Var<xsl:value-of select="." />)</xsl:attribute>
 </xsl:template>
 
 <!-- always returns true -->
@@ -512,22 +507,22 @@ is the second argument -->
 
 <!-- returns true if all the required fields of _event are bound -->
   <xsl:template match="//@conf:eventFieldsAreBound">
-    <xsl:attribute name="cond">'name' in _event &amp;&amp; 'type' in _event &amp;&amp; 'sendid' in _event &amp;&amp; 'origin' in _event &amp;&amp; 'origintype' in _event &amp;&amp; 'invokeid' in _event &amp;&amp; 'data' in _event</xsl:attribute>
+    <xsl:attribute name="cond">empty(_event.getName())?:true and empty(_event.getType())?:true and empty(_event.getSendid())?:true and empty(_event.getOrigin())?:true and empty(_event.getOrigintype())?:true and empty(_event.getInvokeid())?:true and empty(_event.getData())</xsl:attribute>
   </xsl:template> 
 
 <!-- returns true if  _event.data contains the specified item -->
 <xsl:template match="//@conf:eventDataHasField">
-	<xsl:attribute name="cond"><xsl:value-of select="."/> in _event.data</xsl:attribute>
+	<xsl:attribute name="cond">!empty(<xsl:value-of select="."/>_event.data)</xsl:attribute>
 </xsl:template>
 
 <!-- returns true if specified field of _event has no value -->
 <xsl:template match="//@conf:eventFieldHasNoValue">
-	<xsl:attribute name="cond">typeof _event.<xsl:value-of select="." /> === 'undefined' </xsl:attribute>
+	<xsl:attribute name="cond">empty(_event.<xsl:value-of select="." />)</xsl:attribute>
 </xsl:template>
 
 <!-- true if the language of _event matches the processor's datamodel -->
 <xsl:template match="//@conf:eventLanguageMatchesDatamodel">
-	<xsl:attribute name="cond"> _event.language == 'ecmascript'</xsl:attribute>
+	<xsl:attribute name="cond"> _event.language == 'jexl'</xsl:attribute>
 </xsl:template>
 
 <!-- true if _event was delivered on the specified i/o processor -->
@@ -541,7 +536,7 @@ is the second argument -->
 <!-- scripting -->
 
 <xsl:template match="conf:script">
- <script xmlns="http://www.w3.org/2005/07/scxml">var Var1 = 1</script>
+	<script xmlns="http://www.w3.org/2005/07/scxml">Var1 = 1</script>
 </xsl:template>
 
 
@@ -605,7 +600,7 @@ is of the same type as array123 -->
 <xsl:template match="conf:extendArray">
 	<assign xmlns="http://www.w3.org/2005/07/scxml">
 	  <xsl:attribute name="location">Var<xsl:value-of select="@id"/></xsl:attribute>
-	  <xsl:attribute name="expr">[].concat(Var<xsl:value-of select="@id"/>, [4])</xsl:attribute>
+	  <xsl:attribute name="expr">Var<xsl:value-of select="@id"/>.add(4)</xsl:attribute>
 	  </assign>
 	</xsl:template>
 
@@ -685,7 +680,7 @@ the basic http tests.  In the case of python, we have to import the regexp modul
 
 <!-- generate a cond that evaluates to true if the event is external -->
 <xsl:template match="//@conf:eventIsExternal">
- <xsl:attribute name="cond">_event.type === 'external'</xsl:attribute>
+ <xsl:attribute name="cond">_event.type == 'external'</xsl:attribute>
 </xsl:template>
 
 <!-- returns true if _event/raw contains the var with the specified value -->

http://git-wip-us.apache.org/repos/asf/commons-scxml/blob/321fd426/src/test/java/org/apache/commons/scxml2/w3c/tests.xml
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/commons/scxml2/w3c/tests.xml b/src/test/java/org/apache/commons/scxml2/w3c/tests.xml
index 90ee864..deccdac 100644
--- a/src/test/java/org/apache/commons/scxml2/w3c/tests.xml
+++ b/src/test/java/org/apache/commons/scxml2/w3c/tests.xml
@@ -1,203 +1,204 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <tests>
-  <test id="355" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="576" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="364" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="372" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="570" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="375" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="376" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="377" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="378" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="387" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="579" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="580" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="388" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="396" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="399" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="401" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="402" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="403" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="404" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="405" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="406" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="407" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="409" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="411" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="412" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="413" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="415" mandatory="true"  manual="true"  enabled="true"  finalId="final" ecma="final"/>
-  <test id="416" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="417" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="419" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="421" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="422" mandatory="true"  manual="false" enabled="false"                 ecma="fail">fails with error: null</test>
-  <test id="423" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="503" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="504" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="505" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="506" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="533" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="144" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="147" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="148" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="149" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="150" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="151" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="152" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="153" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="155" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="156" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="525" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="158" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="159" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="276" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="277" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="279" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="280" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="550" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="551" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="552" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="286" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="287" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="487" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="294" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="527" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="528" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="529" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="298" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="343" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="488" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="301" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="302" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="303" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="304" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="307" mandatory="true"  manual="true"  enabled="false"                 ecma="fail"/>
-  <test id="309" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="310" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="311" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="312" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="313" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="314" mandatory="true"  manual="true"  enabled="false"                 ecma="fail"/>
-  <test id="344" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="318" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="319" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="321" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="322" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="323" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="324" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="325" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="326" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="329" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="330" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="331" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="332" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="333" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="335" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="336" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="337" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="338" mandatory="true"  manual="false" enabled="false"                 ecma=""    >Fails to complete</test>
-  <test id="339" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="342" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="346" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="172" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="173" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="174" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="175" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="176" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="178" mandatory="true"  manual="true"  enabled="true"  finalId="final" ecma="final"/>
-  <test id="179" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="183" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="185" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="186" mandatory="true"  manual="false" enabled="true"                  ecma="true"/>
-  <test id="187" mandatory="true"  manual="false" enabled="false"                 ecma=""    >fails with error: null</test>
-  <test id="194" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="198" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="199" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="200" mandatory="true"  manual="false" enabled="true"                  ecma="true"/>
-  <test id="201" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="205" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="521" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="553" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="207" mandatory="true"  manual="false" enabled="false"                 ecma=""    >Fails to complete</test>
-  <test id="208" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="210" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="215" mandatory="true"  manual="false" enabled="false"                 ecma=""    >Fails with error: null</test>
-  <test id="216" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="220" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="223" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="224" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="225" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="226" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="228" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="229" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="230" mandatory="true"  manual="true"  enabled="false"                 ecma="fail"/>
-  <test id="232" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="233" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="234" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="235" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="236" mandatory="true"  manual="false" enabled="false"                 ecma=""    >Fails to complete</test>
-  <test id="237" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="239" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="240" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="241" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="242" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="243" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="244" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="245" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="247" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="250" mandatory="true"  manual="true"  enabled="false"                 ecma="fail"/>
-  <test id="252" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="253" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="530" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="554" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="436" mandatory="true"  manual="false" enabled="true"  minimal="pass"/>
-  <test id="278" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="444" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="445" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="448" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="449" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="451" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="452" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="453" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="456" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="446" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="557" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="558" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="560" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="578" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="561" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="562" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="569" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="457" mandatory="false" manual="false" enabled="false"                 ecma="fail"/>
-  <test id="459" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="460" mandatory="false" manual="false" enabled="true"                  ecma="pass"/>
-  <test id="189" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="190" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="191" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="192" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="193" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="347" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="348" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="349" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="350" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="351" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="352" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="354" mandatory="true"  manual="false" enabled="true"                  ecma="pass"/>
-  <test id="495" mandatory="true"  manual="false" enabled="true"                  ecma="true"/>
-  <test id="496" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="500" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="501" mandatory="true"  manual="false" enabled="false"                 ecma="fail"/>
-  <test id="509" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="510" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="513" mandatory="false" manual="true"  enabled="false" implemented="false"/>
-  <test id="518" mandatory="true"  manual="false" enabled="false" implemented="false"/>
-  <test id="519" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="520" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="522" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="531" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="532" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="534" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="567" mandatory="false" manual="false" enabled="false" implemented="false"/>
-  <test id="577" mandatory="false" manual="false" enabled="false" implemented="false"/>
+  <test id="355" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="576" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="364" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="372" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="570" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="375" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="376" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="377" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="378" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="387" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="579" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="580" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="388" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="396" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="399" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="401" mandatory="true"                    manual="false" jexl="false" ecma="true"/>
+  <test id="402" mandatory="true"                    manual="false" jexl="false" ecma="true"/>
+  <test id="403" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="404" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="405" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="406" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="407" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="409" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="411" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="412" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="413" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="415" mandatory="true"                    manual="true"  jexl="true"  ecma="true"  finalState="final"/>
+  <test id="416" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="417" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="419" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="421" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="422" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="423" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="503" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="504" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="505" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="506" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="533" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="144" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="147" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="148" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="149" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="150" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="151" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="152" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="153" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="155" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="156" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="525" mandatory="true"                    manual="false" jexl="true"  ecma="false"/>
+  <test id="158" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="159" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="276" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="277" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="279" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="280" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="550" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="551" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="552" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="286" mandatory="true"                    manual="false" jexl="false" ecma="true"/>
+  <test id="287" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="487" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="294" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="527" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="528" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="529" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="298" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="343" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="488" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="301" mandatory="true"                    manual="true"  jexl="false" ecma="false" finalState="java.net.MalformedURLException"/>
+  <test id="302" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="303" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="304" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="307" mandatory="true"                    manual="true"  jexl="false"  ecma="false" finalState="final"/>
+  <test id="309" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="310" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="311" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="312" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="313" mandatory="true"                    manual="true"  jexl="false" ecma="false" finalState="pass"/>
+  <test id="314" mandatory="true"                    manual="true"  jexl="false" ecma="false" finalState="pass"/>
+  <test id="344" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="318" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="319" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="321" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="322" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="323" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="324" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="325" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="326" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="329" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="330" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="331" mandatory="true"                    manual="false" jexl="false" ecma="true"/>
+  <test id="332" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="333" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="335" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="336" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="337" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="338" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="339" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="342" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="346" mandatory="true"  profile="minimal" manual="false" jexl="true"  ecma="true"/>
+  <test id="172" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="173" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="174" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="175" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="176" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="178" mandatory="true"                    manual="true"  jexl="true"  ecma="true"  finalState="final"/>
+  <test id="179" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="183" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="185" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="186" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="187" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="194" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="198" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="199" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="200" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="205" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="521" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="553" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="207" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="208" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="210" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="215" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="216" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="220" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="223" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="224" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="225" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="226" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="228" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="229" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="230" mandatory="true"                    manual="true"  jexl="false" ecma="false" finalState="final"/>
+  <test id="232" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="233" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="234" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="235" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="236" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="237" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="239" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="240" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="241" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="242" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="243" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="244" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="245" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="247" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="250" mandatory="true"                    manual="true"  jexl="false" ecma="false"  finalState="final"/>
+  <test id="252" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="253" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="530" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="554" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="436" mandatory="true"  profile="minimal" manual="false"                           minimal="true"/>
+  <test id="278" mandatory="false"                   manual="false" jexl="false" ecma="false"/>
+  <test id="444" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="445" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="448" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="449" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="451" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="452" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="453" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="456" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="446" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="557" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="558" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="560" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="578" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="561" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="562" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="569" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="457" mandatory="false" profile="ecma"    manual="false"              ecma="false"/>
+  <test id="459" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="460" mandatory="false" profile="ecma"    manual="false"              ecma="true"/>
+  <test id="189" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="190" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="191" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="192" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="193" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="347" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="348" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="349" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="350" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="351" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="352" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="354" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="495" mandatory="true"                    manual="false" jexl="true"  ecma="true"/>
+  <test id="496" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="500" mandatory="true"                    manual="false" jexl="false" ecma="false"/>
+  <test id="501" mandatory="true"                    manual="false" jexl="true"  ecma="false"/>
+
+  <test id="201" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="509" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="510" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="513" mandatory="false" profile="http io" manual="true"  implemented="false"/>
+  <test id="518" mandatory="true"  profile="http io" manual="false" implemented="false"/>
+  <test id="519" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="520" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="522" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="531" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="532" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="534" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="567" mandatory="false" profile="http io" manual="false" implemented="false"/>
+  <test id="577" mandatory="false" profile="http io" manual="false" implemented="false"/>
 </tests>