You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@netbeans.apache.org by lk...@apache.org on 2023/01/09 07:22:53 UTC
[netbeans] branch master updated: TOML uses ANTLR4 Lexer Support
This is an automated email from the ASF dual-hosted git repository.
lkishalmi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/netbeans.git
The following commit(s) were added to refs/heads/master by this push:
new 88c0c39f20 TOML uses ANTLR4 Lexer Support
88c0c39f20 is described below
commit 88c0c39f203c0815a0547ecbfba4d8d420615a17
Author: Laszlo Kishalmi <la...@gmail.com>
AuthorDate: Sun Jan 8 16:33:29 2023 -0800
TOML uses ANTLR4 Lexer Support
---
ide/languages.toml/nbproject/project.xml | 8 ++
.../languages/toml/LexerInputCharStream.java | 135 ---------------------
.../netbeans/modules/languages/toml/TomlLexer.java | 83 ++-----------
.../modules/languages/toml/TomlTokenId.java | 3 +-
4 files changed, 22 insertions(+), 207 deletions(-)
diff --git a/ide/languages.toml/nbproject/project.xml b/ide/languages.toml/nbproject/project.xml
index ea7fe01d03..309960c9b0 100644
--- a/ide/languages.toml/nbproject/project.xml
+++ b/ide/languages.toml/nbproject/project.xml
@@ -114,6 +114,14 @@
<specification-version>1.80</specification-version>
</run-dependency>
</dependency>
+ <dependency>
+ <code-name-base>org.netbeans.modules.lexer.antlr4</code-name-base>
+ <build-prerequisite/>
+ <compile-dependency/>
+ <run-dependency>
+ <specification-version>1.0</specification-version>
+ </run-dependency>
+ </dependency>
<dependency>
<code-name-base>org.netbeans.modules.parsing.api</code-name-base>
<build-prerequisite/>
diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java
deleted file mode 100644
index 7f2c2877ed..0000000000
--- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/LexerInputCharStream.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.netbeans.modules.languages.toml;
-
-import org.antlr.v4.runtime.CharStream;
-import static org.antlr.v4.runtime.IntStream.UNKNOWN_SOURCE_NAME;
-import org.antlr.v4.runtime.misc.Interval;
-import org.netbeans.spi.lexer.*;
-
-/**
- *
- * @author lkishalmi
- */
-public class LexerInputCharStream implements CharStream {
- private final LexerInput input;
-
- private int tokenMark = Integer.MAX_VALUE;
- private int index = 0;
-
- public LexerInputCharStream(LexerInput input) {
- this.input = input;
- }
-
- @Override
- public String getText(Interval intrvl) {
- if (intrvl.a < tokenMark) {
- throw new UnsupportedOperationException("Can't read before the last token end: " + tokenMark);
- }
- int start = intrvl.a - tokenMark;
- int end = intrvl.b - tokenMark + 1;
- int toread = end - start - input.readLength();
- for (int i = 0; i < toread; i++) {
- input.read();
- }
- String ret = String.valueOf(input.readText(start, end));
- if (toread > 0) {
- input.backup(toread);
- }
- return ret;
- }
-
- @Override
- public void consume() {
- read();
- }
-
- @Override
- public int LA(int count) {
- if (count == 0) {
- return 0; //the behaviour is not defined
- }
-
- int c = 0;
- if (count > 0) {
- for (int i = 0; i < count; i++) {
- c = read();
- }
- backup(count);
- } else {
- backup(count);
- c = read();
- }
- return c;
- }
-
- //Marks are for buffering in ANTLR4, we do not really need them
- @Override
- public int mark() {
- return -1;
- }
-
- public void markToken() {
- tokenMark = index;
- }
-
- @Override
- public void release(int marker) {
- }
-
- @Override
- public int index() {
- return index;
- }
-
- @Override
- public void seek(int i) {
- if (i < index()) {
- backup(index() - i);
- } else {
- while (index() < i) {
- if (read() == LexerInput.EOF) {
- break;
- }
- }
- }
- }
-
-
- private int read() {
- int ret = input.read();
- index += 1;
- return ret;
- }
-
- private void backup(int count) {
- index -= count;
- input.backup(count);
- }
-
- @Override
- public int size() {
- throw new UnsupportedOperationException("Stream size is unknown.");
- }
-
- @Override
- public String getSourceName() {
- return UNKNOWN_SOURCE_NAME;
- }
-}
diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
index 0311f79063..6b2414749c 100644
--- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
+++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java
@@ -20,58 +20,31 @@ package org.netbeans.modules.languages.toml;
import org.antlr.v4.runtime.misc.IntegerStack;
import org.netbeans.api.lexer.Token;
-import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerRestartInfo;
-import org.netbeans.spi.lexer.TokenFactory;
import static org.tomlj.internal.TomlLexer.*;
import static org.netbeans.modules.languages.toml.TomlTokenId.*;
+import org.netbeans.spi.lexer.antlr4.AbstractAntlrLexerBridge;
/**
*
* @author lkishalmi
*/
-public final class TomlLexer implements Lexer<TomlTokenId> {
-
- private final TokenFactory<TomlTokenId> tokenFactory;
- private final org.tomlj.internal.TomlLexer lexer;
- private final LexerInputCharStream input;
-
- public TomlLexer(LexerRestartInfo<TomlTokenId> info) {
- this.tokenFactory = info.tokenFactory();
- this.input = new LexerInputCharStream(info.input());
- try {
- this.lexer = new org.tomlj.internal.TomlLexer(input);
- if (info.state() != null) {
- ((LexerState) info.state()).restore(lexer);
- }
- input.markToken();
- } catch (Throwable ex) {
- ex.printStackTrace();
- throw ex;
- }
- }
+public final class TomlLexer extends AbstractAntlrLexerBridge<org.tomlj.internal.TomlLexer, TomlTokenId> {
- private org.antlr.v4.runtime.Token preFetchedToken = null;
+ public TomlLexer(LexerRestartInfo<TomlTokenId> info, org.tomlj.internal.TomlLexer lexer) {
+ super(info, lexer);
+ }
@Override
- public org.netbeans.api.lexer.Token<TomlTokenId> nextToken() {
- org.antlr.v4.runtime.Token nextToken;
- if (preFetchedToken != null) {
- nextToken = preFetchedToken;
- lexer.getInputStream().seek(preFetchedToken.getStopIndex() + 1);
- preFetchedToken = null;
- } else {
- nextToken = lexer.nextToken();
- }
- int tokenType = nextToken.getType();
- switch (tokenType) {
+ protected Token<TomlTokenId> mapToken(int antlrTokenType) {
+ switch (antlrTokenType) {
case EOF:
return null;
case StringChar:
- return collate(StringChar, STRING);
-
+ return groupToken(STRING, StringChar);
+
case TripleQuotationMark:
case TripleApostrophe:
case QuotationMark:
@@ -133,59 +106,27 @@ public final class TomlLexer implements Lexer<TomlTokenId> {
}
}
- protected org.netbeans.api.lexer.Token<TomlTokenId> collate(int tokenType, TomlTokenId tokenId) {
- preFetchedToken = lexer.nextToken();
- while (preFetchedToken.getType() == tokenType) {
- preFetchedToken = lexer.nextToken();
- }
- lexer.getInputStream().seek(preFetchedToken.getStartIndex());
- return token(tokenId);
- }
-
@Override
public Object state() {
return new LexerState(lexer);
}
- @Override
- public void release() {
- }
-
- private Token<TomlTokenId> token(TomlTokenId id) {
- input.markToken();
- return tokenFactory.createToken(id);
- }
-
- private static class LexerState {
- final int state;
- final int mode;
- final IntegerStack modes;
-
+ private static class LexerState extends AbstractAntlrLexerBridge.LexerState<org.tomlj.internal.TomlLexer> {
final int arrayDepth;
final IntegerStack arrayDepthStack;
LexerState(org.tomlj.internal.TomlLexer lexer) {
- this.state= lexer.getState();
+ super(lexer);
- this.mode = lexer._mode;
- this.modes = new IntegerStack(lexer._modeStack);
this.arrayDepth = lexer.arrayDepth;
this.arrayDepthStack = new IntegerStack(lexer.arrayDepthStack);
}
public void restore(org.tomlj.internal.TomlLexer lexer) {
- lexer.setState(state);
- lexer._modeStack.addAll(modes);
- lexer._mode = mode;
+ super.restore(lexer);
lexer.arrayDepth = arrayDepth;
lexer.arrayDepthStack.addAll(arrayDepthStack);
}
-
- @Override
- public String toString() {
- return String.valueOf(state);
- }
-
}
}
diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java
index 8eef764616..e60e21194e 100644
--- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java
+++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlTokenId.java
@@ -26,6 +26,7 @@ import org.netbeans.api.lexer.TokenId;
import org.netbeans.spi.lexer.LanguageHierarchy;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerRestartInfo;
+import org.netbeans.spi.lexer.antlr4.LexerInputCharStream;
/**
*
@@ -66,7 +67,7 @@ public enum TomlTokenId implements TokenId {
@Override
protected Lexer<TomlTokenId> createLexer(LexerRestartInfo<TomlTokenId> info) {
- return new TomlLexer(info);
+ return new TomlLexer(info, new org.tomlj.internal.TomlLexer(new LexerInputCharStream(info.input())));
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@netbeans.apache.org
For additional commands, e-mail: commits-help@netbeans.apache.org
For further information about the NetBeans mailing lists, visit:
https://cwiki.apache.org/confluence/display/NETBEANS/Mailing+lists