[royale-compiler] branch develop updated: RawMXMLTokenizer: fixed issue where unrecognized characters were ignored instead of being reported a problem (closes #120)

[jo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

joshtynjala pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/royale-compiler.git


The following commit(s) were added to refs/heads/develop by this push:
     new 1e61ee1  RawMXMLTokenizer: fixed issue where unrecognized characters were ignored instead of being reported a problem (closes #120)
1e61ee1 is described below

commit 1e61ee18f3995d3324dd8365edca9fad555465d4
Author: Josh Tynjala <jo...@apache.org>
AuthorDate: Wed Apr 29 14:07:24 2020 -0700

    RawMXMLTokenizer: fixed issue where unrecognized characters were ignored instead of being reported a problem (closes #120)
    
    This brings the behavior of the  MXML parser closer in line with the behavior of the Flex SDK compiler
---
 .../parsing/mxml/BaseRawMXMLTokenizer.java         | 26 ++++++++++++++++++++++
 .../internal/parsing/mxml/RawMXMLTokenizer.lex     |  1 +
 2 files changed, 27 insertions(+)

diff --git a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
index 7fd2f9f..f2bd895 100644
--- a/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
+++ b/compiler/src/main/java/org/apache/royale/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
@@ -22,9 +22,12 @@ package org.apache.royale.compiler.internal.parsing.mxml;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.royale.compiler.common.ISourceLocation;
+import org.apache.royale.compiler.common.SourceLocation;
 import org.apache.royale.compiler.internal.parsing.as.ASToken;
 import org.apache.royale.compiler.parsing.ICMToken;
 import org.apache.royale.compiler.problems.ASDocNotClosedProblem;
+import org.apache.royale.compiler.problems.BadCharacterProblem;
 import org.apache.royale.compiler.problems.CommentNotClosedProblem;
 import org.apache.royale.compiler.problems.ICompilerProblem;
 import org.apache.royale.compiler.problems.MXMLUnclosedTagProblem;
@@ -281,6 +284,29 @@ public abstract class BaseRawMXMLTokenizer
     {
         getProblems().add(new ASDocNotClosedProblem((ASToken)token));
     }
+
+    protected void reportBadCharacterProblem(String badChar)
+    {
+        ISourceLocation location = getCurrentSourceLocation(badChar.length());
+        ICompilerProblem problem = new BadCharacterProblem(location, badChar);
+        getProblems().add(problem);
+    }
+
+    /**
+     * Create a {@code ISourceLocation} object based on the current lexer state.
+     * 
+     * @param tokenLength Length of the problematic input.
+     * @return Current source location used to report a syntax problem.
+     */
+    protected final ISourceLocation getCurrentSourceLocation(int tokenLength)
+    {
+        return new SourceLocation(
+                sourcePath,
+                getOffset(),
+                getOffset() + tokenLength,
+                getLine(),
+                getColumn());
+    }
     
     protected List<ICompilerProblem> problems = null;
 
diff --git a/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex b/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
index be7802b..201f961 100644
--- a/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
+++ b/compiler/src/main/jflex/org/apache/royale/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
@@ -532,4 +532,5 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012]
 
 <YYINITIAL, MARKUP, STRING1, STRING2, MARKUP_IGNORE, STRING1_IGNORE, STRING2_IGNORE> .
 {
+	reportBadCharacterProblem(yytext());
 }