You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@netbeans.apache.org by jl...@apache.org on 2019/04/30 04:48:43 UTC
[netbeans] branch master updated: [NETBEANS-2430] Ensuring the
lexer can be also restarted inside a line,
not only at line boundary. (#1222)
This is an automated email from the ASF dual-hosted git repository.
jlahoda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new 359bf46 [NETBEANS-2430] Ensuring the lexer can be also restarted inside a line, not only at line boundary. (#1222)
359bf46 is described below
commit 359bf461d26b7ab5baceb9a3e07bd0c7d36b16a7
Author: Jan Lahoda <jl...@netbeans.org>
AuthorDate: Tue Apr 30 06:48:36 2019 +0200
[NETBEANS-2430] Ensuring the lexer can be also restarted inside a line, not only at line boundary. (#1222)
---
.../modules/textmate/lexer/TextmateLexer.java | 58 ++++++++++++++++------
.../modules/textmate/lexer/TextmateLexerTest.java | 57 ++++++++++++++++++---
2 files changed, 94 insertions(+), 21 deletions(-)
diff --git a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java
index 168c459..f60a57b 100644
--- a/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java
+++ b/ide/textmate.lexer/src/org/netbeans/modules/textmate/lexer/TextmateLexer.java
@@ -43,32 +43,45 @@ public class TextmateLexer implements Lexer<TextmateTokenId>{
private List<IToken> lineTokens;
private int currentIdx;
private StackElement state;
+ private boolean forceReadLine;
public TextmateLexer(LexerInput li, Object state, TokenFactory<TextmateTokenId> factory, IGrammar grammar) {
- if (state == DO_NOT_RESUME_HERE) {
- throw new IllegalStateException("Invalid resume state!");
- }
this.li = li;
- this.state = (StackElement) state;
this.factory = factory;
this.grammar = grammar;
+ if (state instanceof IntralineState) {
+ IntralineState istate = (IntralineState) state;
+ this.lineLen = istate.lineLen;
+ this.currentOffset = istate.currentOffset;
+ this.lineTokens = istate.lineTokens;
+ this.currentIdx = istate.currentIdx;
+ this.state = istate.state;
+ this.forceReadLine = true;
+ } else {
+ this.state = (StackElement) state;
+ }
}
@Override
public Token<TextmateTokenId> nextToken() {
- if (currentOffset >= lineLen) {
+ if (currentOffset >= lineLen || forceReadLine) {
//read next line:
int read;
while ((read = li.read()) != LexerInput.EOF && read != '\n');
- if (li.readLength() != 0) {
- System.err.println(li.readText().toString());
- lineLen = li.readText().length();
- currentOffset = 0;
- ITokenizeLineResult tokenized = grammar.tokenizeLine(li.readText().toString(), state);
- lineTokens = new ArrayList<>(Arrays.asList(tokenized.getTokens()));
- currentIdx = 0;
- state = tokenized.getRuleStack();
+ if (!forceReadLine) {
+ if (li.readLength() != 0) {
+ System.err.println(li.readText().toString());
+ lineLen = li.readText().length();
+ currentOffset = 0;
+ ITokenizeLineResult tokenized = grammar.tokenizeLine(li.readText().toString(), state);
+ lineTokens = new ArrayList<>(Arrays.asList(tokenized.getTokens()));
+ currentIdx = 0;
+ state = tokenized.getRuleStack();
+ } else {
+ lineTokens = null;
+ }
}
+ forceReadLine = false;
}
if (lineTokens != null && currentIdx < lineTokens.size()) {
IToken current = lineTokens.get(currentIdx);
@@ -101,10 +114,27 @@ public class TextmateLexer implements Lexer<TextmateTokenId>{
@Override
public Object state() {
- return lineLen != currentOffset ? DO_NOT_RESUME_HERE : this.state;
+ return lineLen != currentOffset ? new IntralineState(lineLen, currentOffset, lineTokens, currentIdx, state) : this.state;
}
@Override
public void release() {}
+ private static final class IntralineState {
+ private int lineLen;
+ private int currentOffset;
+ private List<IToken> lineTokens;
+ private int currentIdx;
+ private StackElement state;
+
+ public IntralineState(int lineLen, int currentOffset, List<IToken> lineTokens, int currentIdx, StackElement state) {
+ this.lineLen = lineLen;
+ this.currentOffset = currentOffset;
+ this.lineTokens = lineTokens;
+ this.currentIdx = currentIdx;
+ this.state = state;
+ }
+
+ }
+
}
diff --git a/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java b/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java
index a448cfc..2cb2b7c 100644
--- a/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java
+++ b/ide/textmate.lexer/test/unit/src/org/netbeans/modules/textmate/lexer/TextmateLexerTest.java
@@ -22,9 +22,11 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Arrays;
+import javax.swing.text.BadLocationException;
import javax.swing.text.Document;
import javax.swing.text.PlainDocument;
import org.netbeans.api.lexer.Language;
+import org.netbeans.api.lexer.Token;
import org.netbeans.api.lexer.TokenHierarchy;
import org.netbeans.api.lexer.TokenSequence;
import org.netbeans.junit.NbTestCase;
@@ -116,6 +118,16 @@ public class TextmateLexerTest extends NbTestCase {
assertTokenProperties(ts, "test", "string.test");
LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbbx\n");
assertTokenProperties(ts, "test", "string.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xcccx xaax \n");
+ assertFalse(ts.moveNext());
+ doc.remove(3, 1);
+ ts = hi.tokenSequence();
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
+ assertTokenProperties(ts, "test", "whitespace.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaax");
+ assertTokenProperties(ts, "test", "string.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbbx\n");
+ assertTokenProperties(ts, "test", "string.test");
LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xcccx ");
assertTokenProperties(ts, "test", "string.test");
LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaax");
@@ -126,6 +138,42 @@ public class TextmateLexerTest extends NbTestCase {
assertFalse(ts.moveNext());
}
+ public void testNETBEANS2430() throws Exception {
+ FileObject grammar = FileUtil.createData(FileUtil.getConfigRoot(), "Editors/text/test/grammar.json");
+ try (OutputStream out = grammar.getOutputStream();
+ Writer w = new OutputStreamWriter(out)) {
+ w.write("{ \"scopeName\": \"test\", " +
+ " \"patterns\": [\n" +
+ " { \"name\": \"string.test\",\n" +
+ " \"begin\": \"x([^x]+)x\",\n" +
+ " \"end\": \"x\\\\1x\"\n" +
+ " },\n" +
+ " { \"name\": \"whitespace.test\",\n" +
+ " \"match\": \" +\"\n" +
+ " }\n" +
+ "]}\n");
+ }
+ grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test");
+ Document doc = new PlainDocument();
+ doc.insertString(0, " xaax xbbx\nxccx xaax ", null);
+ doc.putProperty(Language.class, new LanguageHierarchyImpl("text/test", "test").language());
+ TokenHierarchy hi = TokenHierarchy.get(doc);
+ TokenSequence<?> ts = hi.tokenSequence();
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
+ assertTokenProperties(ts, "test", "whitespace.test");
+ //do not fully lex
+ doc.insertString(3, "a", null);
+ ts = hi.tokenSequence();
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
+ assertTokenProperties(ts, "test", "whitespace.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xaaax");
+ assertTokenProperties(ts, "test", "string.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " xbbx\n");
+ assertTokenProperties(ts, "test", "string.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "xccx xaax \n");
+ assertFalse(ts.moveNext());
+ }
+
public void testUTF8() throws Exception {
clearWorkDir();
@@ -141,7 +189,7 @@ public class TextmateLexerTest extends NbTestCase {
}
grammar.setAttribute(LanguageHierarchyImpl.GRAMMAR_MARK, "test");
Document doc = new PlainDocument();
- doc.insertString(0, " večerníček večerníček \n", null);
+ doc.insertString(0, " večerníček večerníček ", null);
doc.putProperty(Language.class, new LanguageHierarchyImpl("text/test", "test").language());
TokenHierarchy<Document> hi = TokenHierarchy.get(doc);
TokenSequence<?> ts = hi.tokenSequence();
@@ -155,12 +203,7 @@ public class TextmateLexerTest extends NbTestCase {
assertTokenProperties(ts, "test", "ident.test");
LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
assertTokenProperties(ts, "test");
- LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
- assertTokenProperties(ts, "test", "whitespace.test");
- LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "večerníček");
- assertTokenProperties(ts, "test", "ident.test");
- LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, " ");
- assertTokenProperties(ts, "test", "whitespace.test");
+ LexerTestUtilities.assertNextTokenEquals(ts, TextmateTokenId.TEXTMATE, "\n");
assertFalse(ts.moveNext());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@netbeans.apache.org
For additional commands, e-mail: commits-help@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists