You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flex.apache.org by ah...@apache.org on 2016/11/23 23:29:32 UTC
git commit: [flex-falcon] [refs/heads/develop] - FLEX-35183
FLEX-35184 fix unterminated CDATA and Comments in MXML and E4X
Repository: flex-falcon
Updated Branches:
refs/heads/develop e28b138d4 -> c040b5c88
FLEX-35183 FLEX-35184 fix unterminated CDATA and Comments in MXML and E4X
Project: http://git-wip-us.apache.org/repos/asf/flex-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/flex-falcon/commit/c040b5c8
Tree: http://git-wip-us.apache.org/repos/asf/flex-falcon/tree/c040b5c8
Diff: http://git-wip-us.apache.org/repos/asf/flex-falcon/diff/c040b5c8
Branch: refs/heads/develop
Commit: c040b5c88980130cd43fb903d52f9cdef59a7101
Parents: e28b138
Author: Alex Harui <ah...@apache.org>
Authored: Wed Nov 23 15:29:10 2016 -0800
Committer: Alex Harui <ah...@apache.org>
Committed: Wed Nov 23 15:29:25 2016 -0800
----------------------------------------------------------------------
.../compiler/internal/caches/MXMLDataCache.java | 2 +
.../internal/parsing/as/BaseRawASTokenizer.java | 22 +++++++
.../parsing/mxml/BaseRawMXMLTokenizer.java | 41 +++++++++++++
.../internal/parsing/mxml/MXMLScopeBuilder.java | 1 +
.../internal/parsing/mxml/MXMLTokenizer.java | 2 +
.../problems/CDataNotClosedProblem.java | 41 +++++++++++++
.../problems/CommentNotClosedProblem.java | 41 +++++++++++++
.../problems/MXMLUnclosedTagProblem.java | 2 +-
.../internal/parsing/as/RawASTokenizer.lex | 48 +++------------
.../internal/parsing/mxml/RawMXMLTokenizer.lex | 64 ++++++--------------
10 files changed, 179 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java b/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
index 7f06a2d..af7f4a2 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/caches/MXMLDataCache.java
@@ -86,6 +86,8 @@ public class MXMLDataCache extends ConcurrentCacheStoreBase<MXMLData>
// Build tags and attributes from the tokens.
final MXMLData mxmlData = new MXMLData(tokens, tokenizer.getPrefixMap(), fileSpec);
+ if (tokenizer.hasTokenizationProblems())
+ mxmlData.getProblems().addAll(tokenizer.getTokenizationProblems());
return mxmlData;
}
catch (FileNotFoundException e)
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
index cf22ce6..3a80b66 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/as/BaseRawASTokenizer.java
@@ -21,6 +21,8 @@ package org.apache.flex.compiler.internal.parsing.as;
import org.apache.flex.compiler.common.ISourceLocation;
import org.apache.flex.compiler.problems.ASDocNotClosedProblem;
+import org.apache.flex.compiler.problems.CDataNotClosedProblem;
+import org.apache.flex.compiler.problems.CommentNotClosedProblem;
import org.apache.flex.compiler.problems.ICompilerProblem;
import org.apache.flex.compiler.problems.StringLiteralMustBeTerminatedBeforeLineBreakProblem;
import org.apache.flex.compiler.problems.StringLiteralNotClosedProblem;
@@ -340,6 +342,26 @@ public abstract class BaseRawASTokenizer extends BaseRawTokenizer<ASToken>
}
/**
+ * Report syntax error: input ended before Comment is closed.
+ */
+ protected final void reportUnclosedComment()
+ {
+ final ISourceLocation location = getCurrentSourceLocation(0);
+ final ICompilerProblem problem = new CommentNotClosedProblem(location);
+ getProblems().add(problem);
+ }
+
+ /**
+ * Report syntax error: input ended before CDATA is closed.
+ */
+ protected final void reportUnclosedCDATA()
+ {
+ final ISourceLocation location = getCurrentSourceLocation(0);
+ final ICompilerProblem problem = new CDataNotClosedProblem(location);
+ getProblems().add(problem);
+ }
+
+ /**
* Convert escaped unicode sequence such as {@code \u00FF} and {@code \xFF}
* to unicode code point.
*
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
index c6709c8..09277cf 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/BaseRawMXMLTokenizer.java
@@ -24,7 +24,10 @@ import java.util.List;
import org.apache.flex.compiler.internal.parsing.as.ASToken;
import org.apache.flex.compiler.parsing.ICMToken;
+import org.apache.flex.compiler.problems.ASDocNotClosedProblem;
+import org.apache.flex.compiler.problems.CommentNotClosedProblem;
import org.apache.flex.compiler.problems.ICompilerProblem;
+import org.apache.flex.compiler.problems.MXMLUnclosedTagProblem;
import antlr.CommonToken;
import antlr.Token;
@@ -59,6 +62,8 @@ public abstract class BaseRawMXMLTokenizer
private Token lastToken = null;
+ protected String sourcePath = null;
+
public BaseRawMXMLTokenizer()
{
// no-arg constructor
@@ -106,6 +111,7 @@ public abstract class BaseRawMXMLTokenizer
CommonToken token = new CommonToken(type, text);
token.setLine(line);
token.setColumn(column);
+ token.setFilename(sourcePath);
lastToken = token;
return token;
}
@@ -159,6 +165,11 @@ public abstract class BaseRawMXMLTokenizer
return lastToken != null ? lastToken.getText() : "";
}
+ public void setSourcePath(String sourcePath)
+ {
+ this.sourcePath = sourcePath;
+ }
+
protected void setLastToken(Token token)
{
lastToken = token;
@@ -241,6 +252,36 @@ public abstract class BaseRawMXMLTokenizer
return yytext() + " (" + line + ")";
}
+ /**
+ * Report an unclosed entity problem
+ *
+ * @param The token
+ */
+ protected void reportUnclosedCDATA(MXMLToken token)
+ {
+ getProblems().add(new MXMLUnclosedTagProblem((ASToken)token, "CDATA"));
+ }
+
+ /**
+ * Report an unclosed entity problem
+ *
+ * @param The token
+ */
+ protected void reportUnclosedComment(MXMLToken token)
+ {
+ getProblems().add(new CommentNotClosedProblem((ASToken)token));
+ }
+
+ /**
+ * Report an unclosed entity problem
+ *
+ * @param The token
+ */
+ protected void reportUnclosedASDocComment(MXMLToken token)
+ {
+ getProblems().add(new ASDocNotClosedProblem((ASToken)token));
+ }
+
protected List<ICompilerProblem> problems = null;
/**
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
index 79e59c2..bd778e7 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLScopeBuilder.java
@@ -106,6 +106,7 @@ public class MXMLScopeBuilder
fileScope.addDefinition(packageDefinition);
problems = new LinkedList<ICompilerProblem>();
+ problems.addAll(mxmlData.getProblems());
includeHandler = new IncludeHandler(fileSpecGetter);
includeHandler.setProjectAndCompilationUnit(project, compilationUnit);
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
index 49a3fea..51718ca 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/internal/parsing/mxml/MXMLTokenizer.java
@@ -89,6 +89,7 @@ public class MXMLTokenizer implements IMXMLTokenizer, Closeable
public MXMLTokenizer(String path)
{
tokenizer = new RawMXMLTokenizer();
+ tokenizer.setSourcePath(path);
problems = new ArrayList<ICompilerProblem>();
rootPrefixMap = new MutablePrefixMap();
this.path = path;
@@ -116,6 +117,7 @@ public class MXMLTokenizer implements IMXMLTokenizer, Closeable
public void setPath(String path) {
this.path = path;
+ tokenizer.setSourcePath(path);
}
public void setReader(Reader reader) {
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java b/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
new file mode 100644
index 0000000..e542f3d
--- /dev/null
+++ b/compiler/src/main/java/org/apache/flex/compiler/problems/CDataNotClosedProblem.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.flex.compiler.problems;
+
+import org.apache.flex.compiler.common.ISourceLocation;
+import org.apache.flex.compiler.problems.annotations.ProblemClassification;
+
+/**
+ * Syntax error: input ended before ASDoc is closed.
+ */
+@ProblemClassification(CompilerProblemClassification.SYNTAX_ERROR)
+public final class CDataNotClosedProblem extends ParserProblem
+{
+ public static final String DESCRIPTION =
+ "input ended before CDATA is closed.";
+
+ public static final int errorCode = 1549;
+
+ public CDataNotClosedProblem(ISourceLocation location)
+ {
+ super(location);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java b/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
new file mode 100644
index 0000000..4c5954c
--- /dev/null
+++ b/compiler/src/main/java/org/apache/flex/compiler/problems/CommentNotClosedProblem.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.flex.compiler.problems;
+
+import org.apache.flex.compiler.common.ISourceLocation;
+import org.apache.flex.compiler.problems.annotations.ProblemClassification;
+
+/**
+ * Syntax error: input ended before ASDoc is closed.
+ */
+@ProblemClassification(CompilerProblemClassification.SYNTAX_ERROR)
+public final class CommentNotClosedProblem extends ParserProblem
+{
+ public static final String DESCRIPTION =
+ "input ended before comment is closed.";
+
+ public static final int errorCode = 1549;
+
+ public CommentNotClosedProblem(ISourceLocation location)
+ {
+ super(location);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
----------------------------------------------------------------------
diff --git a/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java b/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
index dab73ad..b668100 100644
--- a/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
+++ b/compiler/src/main/java/org/apache/flex/compiler/problems/MXMLUnclosedTagProblem.java
@@ -30,7 +30,7 @@ import org.apache.flex.compiler.problems.annotations.ProblemClassification;
public final class MXMLUnclosedTagProblem extends ParserProblem
{
public static final String DESCRIPTION =
- "'${tokenText}' tag (or tag before this tag) is unclosed or tag not allowed here.";
+ "'${tokenText}' tag (or tag before this tag, or non-tag inside this tag) is unclosed or tag not allowed here.";
public static final int errorCode = 1552;
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
----------------------------------------------------------------------
diff --git a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
index f0fd6b6..5c93547 100644
--- a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
+++ b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/as/RawASTokenizer.lex
@@ -988,27 +988,12 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
yybegin(MARKUP_IGNORE);
}
-<CDATA> ([^\]])*
+<CDATA> [^]
{
continueAggregate();
}
-<CDATA> ("]"+[^\]>])
-{
- continueAggregate();
-}
-
-<CDATA> {WHITE_SPACE_CHAR}+
-{
- continueAggregate();
-}
-
-<CDATA> .
-{
- continueAggregate();
-}
-
-<CDATA> ("]""]"+">")
+<CDATA> ~("]""]"+">")
{
continueAggregate();
yybegin(E4X);
@@ -1017,32 +1002,16 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
<CDATA><<EOF>>
{
- continueAggregate();
- yybegin(E4X);
- return buildAggregateToken(TOKEN_E4X_CDATA);
-}
-
-<E4XCOMMENT> ([^-])*
-{
- continueAggregate();
-}
-
-<E4XCOMMENT> ("-"+[^->])
-{
- continueAggregate();
+ reportUnclosedCDATA();
+ return null;
}
-<E4XCOMMENT> {WHITE_SPACE_CHAR}+
+<E4XCOMMENT> [^]
{
continueAggregate();
}
-<E4XCOMMENT> .
-{
- continueAggregate();
-}
-
-<E4XCOMMENT> ("-""-"+">")
+<E4XCOMMENT> ~("-""-"+">")
{
continueAggregate();
yybegin(E4X);
@@ -1051,9 +1020,8 @@ REGEX_CLASS="[" ({REGEX_ESCAPE}|[^\n\r\]\\])* "]"
<E4XCOMMENT><<EOF>>
{
- continueAggregate();
- yybegin(E4X);
- return buildAggregateToken(TOKEN_E4X_COMMENT);
+ reportUnclosedComment();
+ return null;
}
<DIRECTIVE> ([^?])*
http://git-wip-us.apache.org/repos/asf/flex-falcon/blob/c040b5c8/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
----------------------------------------------------------------------
diff --git a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
index 4c3a6f6..4447086 100644
--- a/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
+++ b/compiler/src/main/jflex/org/apache/flex/compiler/internal/parsing/mxml/RawMXMLTokenizer.lex
@@ -81,7 +81,9 @@ public RawMXMLTokenizer()
protected final Token buildToken(int type, int start, int end, int line, int column, String text)
{
- return new MXMLToken(type, start, end, line, column, text);
+ MXMLToken token = new MXMLToken(type, start, end, line, column, text);
+ token.setSourcePath(sourcePath);
+ return token;
}
public void reset()
@@ -260,52 +262,39 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012]
// before returning to the initial state.
//
-<COMMENT,ASDOC_COMMENT> ([^-])*
-{
- continueAggregate();
-}
-
-<COMMENT,ASDOC_COMMENT> ("-"+[^->])
-{
- continueAggregate();
-}
-
-<COMMENT,ASDOC_COMMENT> {WHITE_SPACE_CHAR}+
+<COMMENT,ASDOC_COMMENT> [^]
{
continueAggregate();
}
-<COMMENT> ("-""-"+">")
+<COMMENT> ~("-""-"+">")
{
continueAggregate();
yybegin(YYINITIAL);
return buildAggregateToken(TOKEN_COMMENT);
}
-<ASDOC_COMMENT> ("-""-"+">")
+<ASDOC_COMMENT> ~("-""-"+">")
{
continueAggregate();
yybegin(YYINITIAL);
return buildAggregateToken(TOKEN_ASDOC_COMMENT);
}
-<COMMENT,ASDOC_COMMENT> .
-{
- continueAggregate();
-}
-
<COMMENT><<EOF>>
{
- continueAggregate();
- yybegin(YYINITIAL);
- return buildAggregateToken(TOKEN_COMMENT);
+ MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_COMMENT);
+ if (token != null)
+ reportUnclosedComment(token);
+ return token;
}
<ASDOC_COMMENT><<EOF>>
{
- continueAggregate();
- yybegin(YYINITIAL);
- return buildAggregateToken(TOKEN_ASDOC_COMMENT);
+ MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_ASDOC_COMMENT);
+ if (token != null)
+ reportUnclosedASDocComment(token);
+ return token;
}
//
@@ -517,38 +506,25 @@ WHITE_SPACE_CHAR=[\r\n\ \t\b\012]
// before returning to the initial state.
//
-<CDATA> ([^\]])*
+<CDATA> [^]
{
continueAggregate();
}
-<CDATA> ("]"+[^\]>])
-{
- continueAggregate();
-}
-
-<CDATA> {WHITE_SPACE_CHAR}+
-{
- continueAggregate();
-}
-<CDATA> ("]""]"+">")
+<CDATA> ~("]""]"+">")
{
continueAggregate();
yybegin(YYINITIAL);
return buildAggregateToken(TOKEN_CDATA);
}
-<CDATA> .
-{
- continueAggregate();
-}
-
<CDATA><<EOF>>
{
- continueAggregate();
- yybegin(YYINITIAL);
- return buildAggregateToken(TOKEN_CDATA);
+ MXMLToken token = (MXMLToken)buildAggregateToken(TOKEN_CDATA);
+ if (token != null)
+ reportUnclosedCDATA(token);
+ return token;
}
//