You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2016/12/26 13:33:55 UTC
[01/50] [abbrv] opennlp git commit: Remove references to
PlainTextByLineStream constructor that takes FileChannel [Forced Update!]
Repository: opennlp
Updated Branches:
refs/heads/889 cc2eb5d1c -> 63e969c23 (forced update)
Remove references to PlainTextByLineStream constructor that takes FileChannel
The PlainTextByLineStream that takes FileChannel was deprecated. We can remove safely remove it after reviewing internal code that was still using it.
There still deprecated methods that are using the constructors, but it will handled in another issue. After that we are save to permanently remove the deprecated methods from PlainTextByLineStream.
See issue OPENNLP-882
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c28a5420
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c28a5420
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c28a5420
Branch: refs/heads/889
Commit: c28a5420822f5a704d080c4169434a12fa707a87
Parents: 4a4b591
Author: William Colen <wi...@gmail.com>
Authored: Thu Nov 10 22:58:37 2016 -0200
Committer: William Colen <wi...@gmail.com>
Committed: Thu Nov 10 22:58:37 2016 -0200
----------------------------------------------------------------------
.../DictionaryNameFinderEvaluatorTest.java | 20 ++++++++--------
.../TokenNameFinderCrossValidatorTest.java | 25 ++++++++++----------
2 files changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c28a5420/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java
index 0b5b6ab..2150de7 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/DictionaryNameFinderEvaluatorTest.java
@@ -17,25 +17,26 @@
package opennlp.tools.namefind;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
import static org.junit.Assert.assertTrue;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
+import org.junit.Test;
+
import opennlp.tools.cmdline.namefind.NameEvaluationErrorListener;
import opennlp.tools.dictionary.Dictionary;
+import opennlp.tools.formats.ResourceAsStreamFactory;
+import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.StringList;
import opennlp.tools.util.eval.FMeasure;
-import org.junit.Test;
-
/**
* Tests the evaluation of a {@link DictionaryNameFinder}.
*/
@@ -66,13 +67,12 @@ public class DictionaryNameFinderEvaluatorTest {
*/
private static ObjectStream<NameSample> createSample() throws IOException,
URISyntaxException {
- FileInputStream sampleDataIn = new FileInputStream(new File(
- DictionaryNameFinderEvaluatorTest.class.getClassLoader()
- .getResource("opennlp/tools/namefind/AnnotatedSentences.txt")
- .toURI()));
+
+ InputStreamFactory in = new ResourceAsStreamFactory(
+ DictionaryNameFinderEvaluatorTest.class,
+ "/opennlp/tools/namefind/AnnotatedSentences.txt");
- return new NameSampleDataStream(new PlainTextByLineStream(
- sampleDataIn.getChannel(), "ISO-8859-1"));
+ return new NameSampleDataStream(new PlainTextByLineStream(in, ISO_8859_1));
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c28a5420/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
index 132264e..22567ee 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
@@ -17,23 +17,24 @@
package opennlp.tools.namefind;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
import java.util.Collections;
import java.util.Map;
+import org.junit.Test;
+
import opennlp.tools.cmdline.namefind.NameEvaluationErrorListener;
+import opennlp.tools.formats.ResourceAsStreamFactory;
+import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelType;
-import org.junit.Test;
-
public class TokenNameFinderCrossValidatorTest {
private final String TYPE = null;
@@ -44,12 +45,11 @@ public class TokenNameFinderCrossValidatorTest {
*/
public void testWithNullResources() throws Exception {
- FileInputStream sampleDataIn = new FileInputStream(new File(getClass()
- .getClassLoader()
- .getResource("opennlp/tools/namefind/AnnotatedSentences.txt").toURI()));
+ InputStreamFactory in = new ResourceAsStreamFactory(getClass(),
+ "/opennlp/tools/namefind/AnnotatedSentences.txt");
ObjectStream<NameSample> sampleStream = new NameSampleDataStream(
- new PlainTextByLineStream(sampleDataIn.getChannel(), "ISO-8859-1"));
+ new PlainTextByLineStream(in, ISO_8859_1));
TrainingParameters mlParams = new TrainingParameters();
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
@@ -59,7 +59,7 @@ public class TokenNameFinderCrossValidatorTest {
ModelType.MAXENT.toString());
TokenNameFinderCrossValidator cv = new TokenNameFinderCrossValidator("en",
- TYPE, mlParams, null, null);
+ TYPE, mlParams, null, (TokenNameFinderEvaluationMonitor)null);
cv.evaluate(sampleStream, 2);
@@ -72,12 +72,11 @@ public class TokenNameFinderCrossValidatorTest {
*/
public void testWithNameEvaluationErrorListener() throws Exception {
- FileInputStream sampleDataIn = new FileInputStream(new File(getClass()
- .getClassLoader()
- .getResource("opennlp/tools/namefind/AnnotatedSentences.txt").toURI()));
+ InputStreamFactory in = new ResourceAsStreamFactory(getClass(),
+ "/opennlp/tools/namefind/AnnotatedSentences.txt");
ObjectStream<NameSample> sampleStream = new NameSampleDataStream(
- new PlainTextByLineStream(sampleDataIn.getChannel(), "ISO-8859-1"));
+ new PlainTextByLineStream(in, ISO_8859_1));
TrainingParameters mlParams = new TrainingParameters();
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(70));
[14/50] [abbrv] opennlp git commit: Remove duplicate if in if
Posted by jo...@apache.org.
Remove duplicate if in if
The inner if is identical and will always have
the same result as the outer one.
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/3ab16184
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/3ab16184
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/3ab16184
Branch: refs/heads/889
Commit: 3ab16184657e381a69c3415fed76711147c4e0cf
Parents: cd6b11d
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 19:36:59 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:32 2016 +0100
----------------------------------------------------------------------
.../src/main/java/opennlp/tools/util/model/BaseModel.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/3ab16184/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
index 4a40d15..771958f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
@@ -311,9 +311,7 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
getManifestProperty(SERIALIZER_CLASS_NAME_PREFIX + entryName);
if (artifactSerializerClazzName != null) {
- if (artifactSerializerClazzName != null) {
- factory = ExtensionLoader.instantiateExtension(ArtifactSerializer.class, artifactSerializerClazzName);
- }
+ factory = ExtensionLoader.instantiateExtension(ArtifactSerializer.class, artifactSerializerClazzName);
}
if (factory != null) {
[09/50] [abbrv] opennlp git commit: Replace private text with Leipzig
english news
Posted by jo...@apache.org.
Replace private text with Leipzig english news
See issue OPENNLP-877
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/a7826d2b
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/a7826d2b
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/a7826d2b
Branch: refs/heads/889
Commit: a7826d2b86810ea0e094281c575b99d377021295
Parents: bbc5a34
Author: Joern Kottmann <ko...@gmail.com>
Authored: Thu Dec 15 21:03:28 2016 +0100
Committer: Joern Kottmann <ko...@gmail.com>
Committed: Mon Dec 19 10:36:36 2016 +0100
----------------------------------------------------------------------
.../formats/LeipzigDoccatSampleStream.java | 23 ++-
.../tools/eval/SourceForgeModelEval.java | 169 +++++++++++++------
2 files changed, 136 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/a7826d2b/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java
index 0af66ae..0ac318a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/LeipzigDoccatSampleStream.java
@@ -23,6 +23,7 @@ import java.nio.charset.StandardCharsets;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.tokenize.SimpleTokenizer;
+import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.FilterObjectStream;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.PlainTextByLineStream;
@@ -40,6 +41,8 @@ import opennlp.tools.util.PlainTextByLineStream;
public class LeipzigDoccatSampleStream extends
FilterObjectStream<String, DocumentSample> {
+ private final Tokenizer tokenizer;
+
private final String language;
private final int sentencesPerDocument;
@@ -51,12 +54,26 @@ public class LeipzigDoccatSampleStream extends
* @param in the InputStream pointing to the contents of the sentences.txt input file
* @throws IOException IOException
*/
- LeipzigDoccatSampleStream(String language, int sentencesPerDocument,
- InputStreamFactory in) throws IOException {
+ public LeipzigDoccatSampleStream(String language, int sentencesPerDocument, Tokenizer tokenizer,
+ InputStreamFactory in) throws IOException {
super(new PlainTextByLineStream(in, StandardCharsets.UTF_8));
System.setOut(new PrintStream(System.out, true, "UTF-8"));
this.language = language;
this.sentencesPerDocument = sentencesPerDocument;
+ this.tokenizer = tokenizer;
+ }
+
+ /**
+ * Creates a new LeipzigDoccatSampleStream with the specified parameters.
+ *
+ * @param language the Leipzig input sentences.txt file
+ * @param sentencesPerDocument the number of sentences which should be grouped into once {@link DocumentSample}
+ * @param in the InputStream pointing to the contents of the sentences.txt input file
+ * @throws IOException IOException
+ */
+ public LeipzigDoccatSampleStream(String language, int sentencesPerDocument,
+ InputStreamFactory in) throws IOException {
+ this(language, sentencesPerDocument, SimpleTokenizer.INSTANCE, in);
}
public DocumentSample read() throws IOException {
@@ -68,7 +85,7 @@ public class LeipzigDoccatSampleStream extends
String line;
while (count < sentencesPerDocument && (line = samples.read()) != null) {
- String tokens[] = SimpleTokenizer.INSTANCE.tokenize(line);
+ String tokens[] = tokenizer.tokenize(line);
if (tokens.length == 0) {
throw new IOException("Empty lines are not allowed!");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/a7826d2b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
index f63fcb5..d996afa 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
@@ -22,6 +22,7 @@ import opennlp.tools.chunker.Chunker;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.cmdline.parser.ParserTool;
+import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.formats.LeipzigDoccatSampleStream;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinder;
@@ -55,12 +56,27 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
/**
- * The tests only run if the input text files are available and those
- * are derived from the leipzig corpus.
+ * This tests ensures that the existing SourceForge models perform
+ * like they are expected to.
*
- * Next step is to replace the input texts with ones that don't have license issues.
- * Wikinews is probably a vey good source. In addition also models that
- * can be shared are required to give everyone the possibilty to run this.
+ * To run this tests external the leipzig sentences files is needed:
+ * leipzig/eng_news_2010_300K-sentences.txt, this file can be
+ * obtained from the leipzig corpus project. <br>
+ *
+ * And all the SourceForge models:<br>
+ * - models-sf/en-sent.bin<br>
+ * - models-sf/en-token.bin<br>
+ * - models-sf/en-ner-date.bin<br>
+ * - models-sf/en-ner-location.binn<br>
+ * - models-sf/en-ner-money.bin<br>
+ * - models-sf/en-ner-organization.bin<br>
+ * - models-sf/en-ner-percentage.bi<br>
+ * - models-sf/en-ner-person.bin<br>
+ * - models-sf/en-ner-time.bin<br>
+ * - models-sf/en-chunker.bin<br>
+ * - models-sf/en-pos-maxent.bin<br>
+ * - models-sf/en-pos-perceptron.bin<br>
+ * - models-sf/en-parser-chunking.bin.bin<br>
*/
public class SourceForgeModelEval {
@@ -73,10 +89,28 @@ public class SourceForgeModelEval {
}
@Test
+ public void ensureTestDataIsCorrect() throws IOException {
+ MessageDigest digest = createDigest();
+
+ try (ObjectStream<String> lines = new PlainTextByLineStream(
+ new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
+ "leipzig/eng_news_2010_300K-sentences.txt")), Charset.forName("UTF-8"))) {
+
+ String line;
+ while ((line = lines.read()) != null) {
+ digest.update(line.getBytes("UTF-8"));
+ }
+
+ Assert.assertEquals(new BigInteger("248567841356936801447294643695012852392"),
+ new BigInteger(1, digest.digest()));
+ }
+ }
+
+ @Test
public void evalSentenceModel() throws IOException {
SentenceModel model = new SentenceModel(
- new File("/home/burn/opennlp-data-dir", "models-sf/en-sent.bin"));
+ new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-sent.bin"));
MessageDigest digest = createDigest();
@@ -84,13 +118,16 @@ public class SourceForgeModelEval {
StringBuilder text = new StringBuilder();
- try (ObjectStream<String> lines = new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(new File("/home/burn/opennlp-data-dir",
- "leipzig/sentences.txt")), Charset.forName("UTF-8"))) {
+ try (ObjectStream<DocumentSample> lineBatches = new LeipzigDoccatSampleStream("en", 25,
+ new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
+ "leipzig/eng_news_2010_300K-sentences.txt")))) {
- String line;
- while ((line = lines.read()) != null) {
- text.append(line).append(" ");
+ DocumentSample lineBatch ;
+ while ((lineBatch = lineBatches.read()) != null) {
+ // TODO: Replace with Java 8 join
+ for (String token : lineBatch.getText()) {
+ text.append(token).append(" ");
+ }
}
}
@@ -100,13 +137,17 @@ public class SourceForgeModelEval {
digest.update(sentence.getBytes("UTF-8"));
}
- Assert.assertEquals(new BigInteger("54058993675314170033586747935067060992"),
+ Assert.assertEquals(new BigInteger("228544068397077998410949364710969159291"),
new BigInteger(1, digest.digest()));
}
@Test
public void evalTokenModel() throws IOException {
+ // the input stream is currently tokenized, we should detokenize it again,
+ // (or extend to pass in tokenizer, then whitespace tokenizer can be passed)
+ // and then tokenize it here
+
TokenizerModel model = new TokenizerModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-token.bin"));
@@ -114,23 +155,38 @@ public class SourceForgeModelEval {
Tokenizer tokenizer = new TokenizerME(model);
- try (ObjectStream<String> lines = new PlainTextByLineStream(
+ try (ObjectStream<DocumentSample> lines = new LeipzigDoccatSampleStream("en", 1,
+ WhitespaceTokenizer.INSTANCE,
new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
- "leipzig/sentences.txt")), Charset.forName("UTF-8"))) {
+ "leipzig/eng_news_2010_300K-sentences.txt")))) {
- String line;
+ DocumentSample line;
while ((line = lines.read()) != null) {
- String[] tokens = tokenizer.tokenize(line);
+
+ // TODO: Replace with Java 8 join
+ StringBuffer text = new StringBuffer();
+ for (String token : line.getText()) {
+ text.append(token).append(' ');
+ }
+
+ String[] tokens = tokenizer.tokenize(text.toString());
for (String token : tokens) {
digest.update(token.getBytes("UTF-8"));
}
}
}
- Assert.assertEquals(new BigInteger("309548448163611475251363008574168734058"),
+ Assert.assertEquals(new BigInteger("180602607571756839321060482558626151930"),
new BigInteger(1, digest.digest()));
}
+ private ObjectStream<DocumentSample> createLineWiseStream() throws IOException {
+ return new LeipzigDoccatSampleStream("en", 1,
+ new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
+ "leipzig/eng_news_2010_300K-sentences.txt")));
+ }
+
+
private void evalNameFinder(TokenNameFinderModel model, BigInteger expectedHash)
throws IOException {
@@ -138,13 +194,11 @@ public class SourceForgeModelEval {
TokenNameFinder nameFinder = new NameFinderME(model);
- try (ObjectStream<String> lines = new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(), "leipzig/simpleTok.txt")),
- Charset.forName("UTF-8"))) {
+ try (ObjectStream<DocumentSample> lines = createLineWiseStream()) {
- String line;
+ DocumentSample line;
while ((line = lines.read()) != null) {
- Span[] names = nameFinder.find(WhitespaceTokenizer.INSTANCE.tokenize(line));
+ Span[] names = nameFinder.find(line.getText());
for (Span name : names) {
digest.update((name.getType() + name.getStart() + name.getEnd()).getBytes("UTF-8"));
}
@@ -159,7 +213,7 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-date.bin"));
- evalNameFinder(personModel, new BigInteger("13595680199220579055030594287753821185"));
+ evalNameFinder(personModel, new BigInteger("116570003910213570906062355532299200317"));
}
@Test
@@ -167,7 +221,7 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-location.bin"));
- evalNameFinder(personModel, new BigInteger("61423868331440897441202803979849564658"));
+ evalNameFinder(personModel, new BigInteger("44810593886021404716125849669208680993"));
}
@Test
@@ -175,7 +229,7 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-money.bin"));
- evalNameFinder(personModel, new BigInteger("31779803056581858429003932617173745364"));
+ evalNameFinder(personModel, new BigInteger("65248897509365807977219790824670047287"));
}
@Test
@@ -183,7 +237,7 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-organization.bin"));
- evalNameFinder(personModel, new BigInteger("268615755804346283904103340480818555730"));
+ evalNameFinder(personModel, new BigInteger("50454559690338630659278005157657197233"));
}
@Test
@@ -191,7 +245,7 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-percentage.bin"));
- evalNameFinder(personModel, new BigInteger("1793019183238911248412519564457497503"));
+ evalNameFinder(personModel, new BigInteger("320996882594215344113023719117249515343"));
}
@Test
@@ -207,26 +261,25 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-time.bin"));
- evalNameFinder(personModel, new BigInteger("264798318876255738642952635833268231353"));
+ evalNameFinder(personModel, new BigInteger("282941772380683328816791801782579055940"));
}
@Test
public void evalChunkerModel() throws IOException {
- ChunkerModel model = new ChunkerModel(
- new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-chunker.bin"));
-
MessageDigest digest = createDigest();
- Chunker chunker = new ChunkerME(model);
+ POSTagger tagger = new POSTaggerME(new POSModel(
+ new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-pos-perceptron.bin")));
- try (ObjectStream<String> lines = new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(), "leipzig/simpleTokPos.txt")),
- Charset.forName("UTF-8"))) {
+ Chunker chunker = new ChunkerME(new ChunkerModel(
+ new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-chunker.bin")));
- String line;
+ try (ObjectStream<DocumentSample> lines = createLineWiseStream()) {
+
+ DocumentSample line;
while ((line = lines.read()) != null) {
- POSSample sentence = POSSample.parse(line);
+ POSSample sentence = new POSSample(line.getText(), tagger.tag(line.getText()));
String[] chunks = chunker.chunk(sentence.getSentence(), sentence.getTags());
for (String chunk : chunks) {
@@ -235,22 +288,24 @@ public class SourceForgeModelEval {
}
}
- Assert.assertEquals(new BigInteger("87766988424222321513554054789708059330"),
+ Assert.assertEquals(new BigInteger("226003515785585284478071030961407561943"),
new BigInteger(1, digest.digest()));
}
private void evalPosModel(POSModel model, BigInteger expectedHash) throws IOException {
+
+ // break the input stream into sentences
+ // The input stream is tokenized and can be processed here directly
+
MessageDigest digest = createDigest();
POSTagger tagger = new POSTaggerME(model);
- try (ObjectStream<String> lines = new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(new File(EvalUtil.getOpennlpDataDir(),
- "leipzig/simpleTok.txt")), Charset.forName("UTF-8"))) {
+ try (ObjectStream<DocumentSample> lines = createLineWiseStream()) {
- String line;
+ DocumentSample line;
while ((line = lines.read()) != null) {
- String[] tags = tagger.tag(WhitespaceTokenizer.INSTANCE.tokenize(line));
+ String[] tags = tagger.tag(line.getText());
for (String tag : tags) {
digest.update(tag.getBytes("UTF-8"));
}
@@ -265,7 +320,7 @@ public class SourceForgeModelEval {
POSModel maxentModel = new POSModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-pos-maxent.bin"));
- evalPosModel(maxentModel, new BigInteger("6912278014292642909634347798602234960"));
+ evalPosModel(maxentModel, new BigInteger("231995214522232523777090597594904492687"));
}
@Test
@@ -273,28 +328,36 @@ public class SourceForgeModelEval {
POSModel perceptronModel = new POSModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-pos-perceptron.bin"));
- evalPosModel(perceptronModel, new BigInteger("333081688760132868394207450128996236484"));
+ evalPosModel(perceptronModel, new BigInteger("209440430718727101220960491543652921728"));
}
@Test
public void evalParserModel() throws IOException {
+ // break input stream into sentences
+ // input is tokenized already
+
ParserModel model = new ParserModel(
- new File("/home/burn/opennlp-data-dir", "models-sf/en-parser-chunking.bin"));
+ new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-parser-chunking.bin"));
MessageDigest digest = createDigest();
Parser parser = ParserFactory.create(model);
- try (ObjectStream<String> lines = new PlainTextByLineStream(
- new MarkableFileInputStreamFactory(new File("/home/burn/opennlp-data-dir",
- "leipzig/simpleTok.txt")), Charset.forName("UTF-8"))) {
+ try (ObjectStream<DocumentSample> lines = createLineWiseStream()) {
- String line;
+ DocumentSample line;
while ((line = lines.read()) != null) {
- Parse[] parse = ParserTool.parseLine(line, parser, 1);
+ StringBuilder textLine = new StringBuilder();
+
+ // TODO: Replace with Java 8 join
+ for (String token : line.getText()) {
+ textLine.append(token).append(' ');
+ }
+
+ Parse[] parse = ParserTool.parseLine(textLine.toString(), parser, 1);
if (parse.length > 0) {
digest.update(parse[0].toString().getBytes("UTF-8"));
}
@@ -304,7 +367,7 @@ public class SourceForgeModelEval {
}
}
- Assert.assertEquals(new BigInteger("95566096874728850374427554294889512256"),
+ Assert.assertEquals(new BigInteger("226003515785585284478071030961407561943"),
new BigInteger(1, digest.digest()));
}
}
[33/50] [abbrv] opennlp git commit: Update version from 1.6.1 to 1.7.0
Posted by jo...@apache.org.
Update version from 1.6.1 to 1.7.0
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/f89f3ffe
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/f89f3ffe
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/f89f3ffe
Branch: refs/heads/889
Commit: f89f3ffe2121acc223631c71ea5bdcf6764194a7
Parents: 5f36766
Author: J�rn Kottmann <jo...@apache.org>
Authored: Tue Dec 20 20:29:42 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Tue Dec 20 20:29:42 2016 +0100
----------------------------------------------------------------------
opennlp-brat-annotator/pom.xml | 2 +-
opennlp-distr/pom.xml | 2 +-
opennlp-docs/pom.xml | 2 +-
opennlp-morfologik-addon/pom.xml | 2 +-
opennlp-tools/pom.xml | 2 +-
opennlp-uima/pom.xml | 2 +-
pom.xml | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/opennlp-brat-annotator/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-brat-annotator/pom.xml b/opennlp-brat-annotator/pom.xml
index 5f77bb7..65e3b69 100644
--- a/opennlp-brat-annotator/pom.xml
+++ b/opennlp-brat-annotator/pom.xml
@@ -18,7 +18,7 @@
<parent>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/opennlp-distr/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-distr/pom.xml b/opennlp-distr/pom.xml
index e5243cf..13ee88f 100644
--- a/opennlp-distr/pom.xml
+++ b/opennlp-distr/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/opennlp-docs/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-docs/pom.xml b/opennlp-docs/pom.xml
index 29a26eb..18c0b3e 100644
--- a/opennlp-docs/pom.xml
+++ b/opennlp-docs/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/opennlp-morfologik-addon/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/pom.xml b/opennlp-morfologik-addon/pom.xml
index da9b728..ddfedf2 100644
--- a/opennlp-morfologik-addon/pom.xml
+++ b/opennlp-morfologik-addon/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/opennlp-tools/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-tools/pom.xml b/opennlp-tools/pom.xml
index 9afaff4..9e86eb0 100644
--- a/opennlp-tools/pom.xml
+++ b/opennlp-tools/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/opennlp-uima/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/pom.xml b/opennlp-uima/pom.xml
index 11bd95f..cf95b17 100644
--- a/opennlp-uima/pom.xml
+++ b/opennlp-uima/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f89f3ffe/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 89d0206..df60712 100644
--- a/pom.xml
+++ b/pom.xml
@@ -31,7 +31,7 @@
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp</artifactId>
- <version>1.6.1-SNAPSHOT</version>
+ <version>1.7.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache OpenNLP Reactor</name>
[21/50] [abbrv] opennlp git commit: Use StringBuilder.append and not
string cat
Posted by jo...@apache.org.
Use StringBuilder.append and not string cat
The argument to StringBuilder.append should not be
concatenated. Instead StringBuilder.append can be called
more often.
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/7ef7235f
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/7ef7235f
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/7ef7235f
Branch: refs/heads/889
Commit: 7ef7235f04f9e26cf41155a92ab2a2632a7ba7e6
Parents: b854d31
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 19:52:08 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../java/opennlp/tools/cmdline/DetailedFMeasureListener.java | 6 +++---
.../main/java/opennlp/tools/cmdline/GenerateManualTool.java | 8 ++++----
.../tools/formats/ontonotes/OntoNotesParseSampleStream.java | 2 +-
3 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/7ef7235f/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
index f4c38cb..ccf6884 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
@@ -120,9 +120,9 @@ public abstract class DetailedFMeasureListener<T> implements
StringBuilder ret = new StringBuilder();
int tp = generalStats.getTruePositives();
int found = generalStats.getFalsePositives() + tp;
- ret.append("Evaluated " + samples + " samples with "
- + generalStats.getTarget() + " entities; found: " + found
- + " entities; correct: " + tp + ".\n");
+ ret.append("Evaluated ").append(samples).append(" samples with ")
+ .append(generalStats.getTarget()).append(" entities; found: ")
+ .append(found).append(" entities; correct: ").append(tp).append(".\n");
ret.append(String.format(locale, FORMAT, "TOTAL",
zeroOrPositive(generalStats.getPrecisionScore() * 100),
http://git-wip-us.apache.org/repos/asf/opennlp/blob/7ef7235f/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
index b9dcc2d..ed58dde 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
@@ -160,8 +160,8 @@ public class GenerateManualTool {
for (Argument argument : arguments) {
sb.append("<row>\n");
if (i == 0) {
- sb.append("<entry morerows='" + (arguments.size() - 1)
- + "' valign='middle'>").append(format).append("</entry>\n");
+ sb.append("<entry morerows='").append(arguments.size() - 1)
+ .append("' valign='middle'>").append(format).append("</entry>\n");
}
sb.append("<entry>").append(argument.getArgument())
.append("</entry>\n");
@@ -229,8 +229,8 @@ public class GenerateManualTool {
}
private static void appendCode(String help, StringBuilder sb) {
- sb.append("<screen>\n" + "<![CDATA[\n" + splitLongLines(help) + "\n"
- + "]]>\n" + "</screen> \n");
+ sb.append("<screen>\n" + "<![CDATA[\n").append(splitLongLines(help))
+ .append("\n").append("]]>\n").append("</screen> \n");
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/7ef7235f/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
index 690196a..1627ca5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStream.java
@@ -51,7 +51,7 @@ public class OntoNotesParseSampleStream extends FilterObjectStream<String, Parse
}
}
- parseString.append(parse + " ");
+ parseString.append(parse).append(" ");
}
}
}
[05/50] [abbrv] opennlp git commit: Remove deprecated API from the
Parser
Posted by jo...@apache.org.
Remove deprecated API from the Parser
See issue OPENNLP-886
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/9c140512
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/9c140512
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/9c140512
Branch: refs/heads/889
Commit: 9c140512d9caadda9ea469b679766870304ce947
Parents: 639b9f0
Author: J�rn Kottmann <jo...@apache.org>
Authored: Tue Nov 22 01:58:01 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Tue Nov 22 01:58:01 2016 +0100
----------------------------------------------------------------------
.../opennlp/tools/parser/chunking/Parser.java | 32 --------------------
1 file changed, 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9c140512/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
index 044477b..77a030f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
@@ -248,15 +248,6 @@ public class Parser extends AbstractBottomUpParser {
return newParses;
}
- /**
- * @deprecated Please do not use anymore, use the ObjectStream train methods instead! This method
- * will be removed soon.
- */
- @Deprecated
- public static AbstractModel train(ObjectStream<Event> es, int iterations, int cut) throws java.io.IOException {
- return opennlp.tools.ml.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
- }
-
public static void mergeReportIntoManifest(Map<String, String> manifest,
Map<String, String> report, String namespace) {
@@ -318,27 +309,4 @@ public class Parser extends AbstractBottomUpParser {
posModel, chunkModel, (opennlp.tools.parser.HeadRules) rules,
ParserType.CHUNKING, manifestInfoEntries);
}
-
- /**
- * @deprecated use {@link #train(String, ObjectStream, HeadRules, TrainingParameters)}
- * instead and pass in a TrainingParameters object.
- */
- @Deprecated
- public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut)
- throws IOException {
-
- TrainingParameters params = new TrainingParameters();
- params.put("dict", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
-
- params.put("tagger", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("tagger", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put("chunker", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("chunker", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put("check", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("check", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put("build", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("build", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
-
- return train(languageCode, parseSamples, rules, params);
- }
}
[20/50] [abbrv] opennlp git commit: Remove redundant array creation
Posted by jo...@apache.org.
Remove redundant array creation
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c4798464
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c4798464
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c4798464
Branch: refs/heads/889
Commit: c479846405b5ea919609d2e63e25f3048c20d31e
Parents: 3ab1618
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 19:45:47 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../tools/namefind/DefaultNameContextGenerator.java | 12 +++++-------
.../java/opennlp/tools/namefind/NameFinderME.java | 14 ++++++--------
2 files changed, 11 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c4798464/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java
index 6fe15ed..8dcfe1d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/DefaultNameContextGenerator.java
@@ -40,13 +40,11 @@ public class DefaultNameContextGenerator implements NameContextGenerator {
@Deprecated
private static AdaptiveFeatureGenerator windowFeatures = new CachedFeatureGenerator(
- new AdaptiveFeatureGenerator[]{
- new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
- new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
- new OutcomePriorFeatureGenerator(),
- new PreviousMapFeatureGenerator(),
- new BigramNameFeatureGenerator()
- });
+ new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
+ new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
+ new OutcomePriorFeatureGenerator(),
+ new PreviousMapFeatureGenerator(),
+ new BigramNameFeatureGenerator());
/**
* Creates a name context generator.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c4798464/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
index df06f2a..1824909 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
@@ -101,14 +101,12 @@ public class NameFinderME implements TokenNameFinder {
*/
static AdaptiveFeatureGenerator createFeatureGenerator() {
return new CachedFeatureGenerator(
- new AdaptiveFeatureGenerator[]{
- new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
- new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
- new OutcomePriorFeatureGenerator(),
- new PreviousMapFeatureGenerator(),
- new BigramNameFeatureGenerator(),
- new SentenceFeatureGenerator(true, false)
- });
+ new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
+ new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
+ new OutcomePriorFeatureGenerator(),
+ new PreviousMapFeatureGenerator(),
+ new BigramNameFeatureGenerator(),
+ new SentenceFeatureGenerator(true, false));
}
private static AdaptiveFeatureGenerator createFeatureGenerator(
[19/50] [abbrv] opennlp git commit: Remove calls to String.toString()
Posted by jo...@apache.org.
Remove calls to String.toString()
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/b854d314
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/b854d314
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/b854d314
Branch: refs/heads/889
Commit: b854d3142b718aff8f44077bbbb1905dc455364d
Parents: c479846
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 19:47:18 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../tools/lemmatizer/DefaultLemmatizerContextGenerator.java | 2 +-
.../java/opennlp/tools/postag/DefaultPOSSequenceValidator.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/b854d314/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
index c271ab4..e455f01 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
@@ -67,7 +67,7 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener
// Previous prediction
String p_1;
- String lex = toks[index].toString();
+ String lex = toks[index];
if (index < 1) {
p_1 = "p_1=bos";
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/b854d314/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
index 56530ae..0d82e0b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
@@ -34,7 +34,7 @@ public class DefaultPOSSequenceValidator implements SequenceValidator<String> {
if (tagDictionary == null) {
return true;
} else {
- String[] tags = tagDictionary.getTags(inputSequence[i].toString());
+ String[] tags = tagDictionary.getTags(inputSequence[i]);
if (tags == null) {
return true;
} else {
[48/50] [abbrv] opennlp git commit: OPENNLP-870: ContextGenerator is
now Generic
Posted by jo...@apache.org.
OPENNLP-870: ContextGenerator is now Generic
This closes #19
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/f781fcaf
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/f781fcaf
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/f781fcaf
Branch: refs/heads/889
Commit: f781fcaf0f19766f7c9105911613f86d22b2091a
Parents: 2210868
Author: Daniel Russ <dr...@mail.nih.gov>
Authored: Fri Dec 23 14:35:12 2016 -0500
Committer: Kottmann <jo...@apache.org>
Committed: Sun Dec 25 16:36:49 2016 +0100
----------------------------------------------------------------------
.../main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java | 4 ++--
.../src/main/java/opennlp/tools/ml/maxent/ContextGenerator.java | 4 ++--
.../main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f781fcaf/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
index 8cc39cc..471fe63 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
@@ -29,7 +29,7 @@ package opennlp.tools.ml.maxent;
* cp_1 cp_2 ... cp_n
* </p>
*/
-public class BasicContextGenerator implements ContextGenerator {
+public class BasicContextGenerator implements ContextGenerator<String> {
private String separator = " ";
@@ -42,7 +42,7 @@ public class BasicContextGenerator implements ContextGenerator {
/**
* Builds up the list of contextual predicates given a String.
*/
- public String[] getContext(Object o) {
+ public String[] getContext(String o) {
String s = (String) o;
return s.split(separator);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f781fcaf/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/ContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/ContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/ContextGenerator.java
index 0582323..fa92846 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/ContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/ContextGenerator.java
@@ -22,11 +22,11 @@ package opennlp.tools.ml.maxent;
/**
* Generate contexts for maxent decisions.
*/
-public interface ContextGenerator {
+public interface ContextGenerator<T> {
/**
* Builds up the list of contextual predicates given an Object.
*/
- public String[] getContext(Object o);
+ public String[] getContext(T o);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/f781fcaf/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
index f2c3b82..462cf96 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
@@ -26,7 +26,7 @@ import opennlp.tools.ml.model.RealValueFileEventStream;
import opennlp.tools.util.ObjectStream;
public class RealBasicEventStream implements ObjectStream<Event> {
- ContextGenerator cg = new BasicContextGenerator();
+ ContextGenerator<String> cg = new BasicContextGenerator();
private ObjectStream<String> ds;
public RealBasicEventStream(ObjectStream<String> ds) {
[11/50] [abbrv] opennlp git commit: OPENNLP-887: Replace the Cache
class with a LinkedHashMap
Posted by jo...@apache.org.
OPENNLP-887: Replace the Cache class with a LinkedHashMap
This closes #9
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/4da7f4c6
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/4da7f4c6
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/4da7f4c6
Branch: refs/heads/889
Commit: 4da7f4c643d45b57a6a94e34797481b476194336
Parents: 7d1123a
Author: smarthi <sm...@apache.org>
Authored: Mon Dec 19 12:51:54 2016 -0500
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 22:16:53 2016 +0100
----------------------------------------------------------------------
.../main/java/opennlp/tools/ml/BeamSearch.java | 10 +-
.../opennlp/tools/ngram/NGramGenerator.java | 4 +-
.../tools/parser/ChunkContextGenerator.java | 23 +-
.../postag/DefaultPOSContextGenerator.java | 15 +-
.../java/opennlp/tools/util/BeamSearch.java | 22 +-
.../src/main/java/opennlp/tools/util/Cache.java | 329 +------------------
.../util/featuregen/CachedFeatureGenerator.java | 8 +-
7 files changed, 42 insertions(+), 369 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java b/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
index 0ed5fe6..209d4af 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/BeamSearch.java
@@ -47,7 +47,7 @@ public class BeamSearch<T> implements SequenceClassificationModel<T> {
protected MaxentModel model;
private double[] probs;
- private Cache contextsCache;
+ private Cache<String[], double[]> contextsCache;
private static final int zeroLog = -100000;
/**
@@ -66,7 +66,7 @@ public class BeamSearch<T> implements SequenceClassificationModel<T> {
this.model = model;
if (cacheSize > 0) {
- contextsCache = new Cache(cacheSize);
+ contextsCache = new Cache<>(cacheSize);
}
this.probs = new double[model.getNumOutcomes()];
@@ -102,7 +102,7 @@ public class BeamSearch<T> implements SequenceClassificationModel<T> {
String[] contexts = cg.getContext(i, sequence, outcomes, additionalContext);
double[] scores;
if (contextsCache != null) {
- scores = (double[]) contextsCache.get(contexts);
+ scores = contextsCache.get(contexts);
if (scores == null) {
scores = model.eval(contexts, probs);
contextsCache.put(contexts,scores);
@@ -113,9 +113,7 @@ public class BeamSearch<T> implements SequenceClassificationModel<T> {
}
double[] temp_scores = new double[scores.length];
- for (int c = 0; c < scores.length; c++) {
- temp_scores[c] = scores[c];
- }
+ System.arraycopy(scores, 0, temp_scores, 0, scores.length);
Arrays.sort(temp_scores);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
index 7e05a93..f001ba2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramGenerator.java
@@ -37,7 +37,7 @@ public class NGramGenerator {
*/
public static List<String> generate(List<String> input, int n, String separator) {
- List<String> outGrams = new ArrayList<String>();
+ List<String> outGrams = new ArrayList<>();
for (int i = 0; i < input.size() - (n - 2); i++) {
String gram = "";
if ((i + n) <= input.size()) {
@@ -59,7 +59,7 @@ public class NGramGenerator {
*/
public static List<String> generate(char[] input, int n, String separator) {
- List<String> outGrams = new ArrayList<String>();
+ List<String> outGrams = new ArrayList<>();
for (int i = 0; i < input.length - (n - 2); i++) {
String gram = "";
if ((i + n) <= input.length) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
index 3619c57..7471b3c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
@@ -29,7 +29,7 @@ import opennlp.tools.util.Cache;
public class ChunkContextGenerator implements ChunkerContextGenerator {
private static final String EOS = "eos";
- private Cache contextsCache;
+ private Cache<String, String[]> contextsCache;
private Object wordsKey;
@@ -40,13 +40,13 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
public ChunkContextGenerator(int cacheSize) {
super();
if (cacheSize > 0) {
- contextsCache = new Cache(cacheSize);
+ contextsCache = new Cache<>(cacheSize);
}
}
public String[] getContext(Object o) {
Object[] data = (Object[]) o;
- return getContext(((Integer) data[0]).intValue(), (String[]) data[1], (String[]) data[2], (String[]) data[3]);
+ return getContext((Integer) data[0], (String[]) data[1], (String[]) data[2], (String[]) data[3]);
}
public String[] getContext(int i, String[] words, String[] prevDecisions, Object[] ac) {
@@ -54,12 +54,11 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
}
public String[] getContext(int i, String[] words, String[] tags, String[] preds) {
- List<String> features = new ArrayList<String>(19);
- int x0 = i;
- int x_2 = x0 - 2;
- int x_1 = x0 - 1;
- int x2 = x0 + 2;
- int x1 = x0 + 1;
+ List<String> features = new ArrayList<>(19);
+ int x_2 = i - 2;
+ int x_1 = i - 1;
+ int x2 = i + 2;
+ int x1 = i + 1;
String w_2,w_1,w0,w1,w2;
String t_2,t_1,t0,t1,t2;
@@ -90,8 +89,8 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
}
// chunkandpostag(0)
- t0=tags[x0];
- w0=words[x0];
+ t0=tags[i];
+ w0=words[i];
// chunkandpostag(1)
if (x1 < tags.length) {
@@ -113,7 +112,7 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
w2=EOS;
}
- String cacheKey = x0+t_2+t1+t0+t1+t2+p_2+p_1;
+ String cacheKey = i +t_2+t1+t0+t1+t2+p_2+p_1;
if (contextsCache!= null) {
if (wordsKey == words) {
String[] contexts = (String[]) contextsCache.get(cacheKey);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
index e570c89..581fed5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSContextGenerator.java
@@ -39,7 +39,7 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
private static Pattern hasCap = Pattern.compile("[A-Z]");
private static Pattern hasNum = Pattern.compile("[0-9]");
- private Cache contextsCache;
+ private Cache<String, String[]> contextsCache;
private Object wordsKey;
private Dictionary dict;
@@ -64,12 +64,12 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
this.dict = dict;
dictGram = new String[1];
if (cacheSize > 0) {
- contextsCache = new Cache(cacheSize);
+ contextsCache = new Cache<>(cacheSize);
}
}
protected static String[] getPrefixes(String lex) {
String[] prefs = new String[PREFIX_LENGTH];
- for (int li = 0, ll = PREFIX_LENGTH; li < ll; li++) {
+ for (int li = 0; li < PREFIX_LENGTH; li++) {
prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
}
return prefs;
@@ -77,7 +77,7 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
protected static String[] getSuffixes(String lex) {
String[] suffs = new String[SUFFIX_LENGTH];
- for (int li = 0, ll = SUFFIX_LENGTH; li < ll; li++) {
+ for (int li = 0; li < SUFFIX_LENGTH; li++) {
suffs[li] = lex.substring(Math.max(lex.length() - li - 1, 0));
}
return suffs;
@@ -95,10 +95,9 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
* @return The context for making a pos tag decision at the specified token index given the specified tokens and previous tags.
*/
public String[] getContext(int index, Object[] tokens, String[] tags) {
- String next, nextnext, lex, prev, prevprev;
+ String next, nextnext = null, lex, prev, prevprev = null;
String tagprev, tagprevprev;
tagprev = tagprevprev = null;
- next = nextnext = lex = prev = prevprev = null;
lex = tokens[index].toString();
if (tokens.length > index + 1) {
@@ -131,7 +130,7 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
String cacheKey = index+tagprev+tagprevprev;
if (contextsCache != null) {
if (wordsKey == tokens){
- String[] cachedContexts = (String[]) contextsCache.get(cacheKey);
+ String[] cachedContexts = contextsCache.get(cacheKey);
if (cachedContexts != null) {
return cachedContexts;
}
@@ -141,7 +140,7 @@ public class DefaultPOSContextGenerator implements POSContextGenerator {
wordsKey = tokens;
}
}
- List<String> e = new ArrayList<String>();
+ List<String> e = new ArrayList<>();
e.add("default");
// add the word itself
e.add("w=" + lex);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java b/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
index 8c460ea..95cbea9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearch.java
@@ -44,7 +44,7 @@ public class BeamSearch<T> {
private SequenceValidator<T> validator;
private double[] probs;
- private Cache contextsCache;
+ private Cache<String[], double[]> contextsCache;
private static final int zeroLog = -100000;
/**
@@ -72,7 +72,7 @@ public class BeamSearch<T> {
this.validator = validator;
if (cacheSize > 0) {
- contextsCache = new Cache(cacheSize);
+ contextsCache = new Cache<>(cacheSize);
}
this.probs = new double[model.getNumOutcomes()];
@@ -86,13 +86,7 @@ public class BeamSearch<T> {
* @see SequenceValidator
*/
private boolean validSequence(int i, T[] inputSequence, String[] outcomesSequence, String outcome) {
-
- if (validator != null) {
- return validator.validSequence(i, inputSequence, outcomesSequence, outcome);
- }
- else {
- return true;
- }
+ return validator == null || validator.validSequence(i, inputSequence, outcomesSequence, outcome);
}
public Sequence[] bestSequences(int numSequences, T[] sequence, Object[] additionalContext) {
@@ -110,8 +104,8 @@ public class BeamSearch<T> {
*/
public Sequence[] bestSequences(int numSequences, T[] sequence, Object[] additionalContext, double minSequenceScore) {
- Heap<Sequence> prev = new ListHeap<Sequence>(size);
- Heap<Sequence> next = new ListHeap<Sequence>(size);
+ Heap<Sequence> prev = new ListHeap<>(size);
+ Heap<Sequence> next = new ListHeap<>(size);
Heap<Sequence> tmp;
prev.add(new Sequence());
@@ -129,7 +123,7 @@ public class BeamSearch<T> {
String[] contexts = cg.getContext(i, sequence, outcomes, additionalContext);
double[] scores;
if (contextsCache != null) {
- scores = (double[]) contextsCache.get(contexts);
+ scores = contextsCache.get(contexts);
if (scores == null) {
scores = model.eval(contexts, probs);
contextsCache.put(contexts,scores);
@@ -140,9 +134,7 @@ public class BeamSearch<T> {
}
double[] temp_scores = new double[scores.length];
- for (int c = 0; c < scores.length; c++) {
- temp_scores[c] = scores[c];
- }
+ System.arraycopy(scores, 0, temp_scores, 0, scores.length);
Arrays.sort(temp_scores);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/util/Cache.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/Cache.java b/opennlp-tools/src/main/java/opennlp/tools/util/Cache.java
index 5ae82fd..fb4d6cb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/Cache.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/Cache.java
@@ -17,337 +17,22 @@
package opennlp.tools.util;
-import java.util.Collection;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.Set;
/**
* Provides fixed size, pre-allocated, least recently used replacement cache.
*/
-@SuppressWarnings("unchecked")
-public class Cache implements Map {
+public class Cache<K,V> extends LinkedHashMap<K,V> {
- /** The element in the linked list which was most recently used. **/
- private DoubleLinkedListElement first;
- /** The element in the linked list which was least recently used. **/
- private DoubleLinkedListElement last;
- /** Temporary holder of the key of the least-recently-used element. */
- private Object lastKey;
- /** Temporary value used in swap. */
- private ObjectWrapper temp;
- /** Holds the object wrappers which the keys are mapped to. */
- private ObjectWrapper[] wrappers;
- /** Map which stores the keys and values of the cache. */
- private Map map;
- /** The size of the cache. */
- private int size;
+ private int capacity;
- /**
- * Creates a new cache of the specified size.
- * @param size The size of the cache.
- */
- public Cache(int size) {
- map = new HashMap(size);
- wrappers = new ObjectWrapper[size];
- this.size=size;
- Object o = new Object();
- first = new DoubleLinkedListElement(null, null, o);
- map.put(o, new ObjectWrapper(null, first));
- wrappers[0] = new ObjectWrapper(null, first);
-
- DoubleLinkedListElement e = first;
- for(int i=1; i<size; i++) {
- o = new Object();
- e = new DoubleLinkedListElement(e, null, o);
- wrappers[i] = new ObjectWrapper(null, e);
- map.put(o, wrappers[i]);
- e.prev.next = e;
- }
- last = e;
- }
-
- public void clear() {
- map.clear();
- DoubleLinkedListElement e = first;
- for (int oi=0;oi<size;oi++) {
- wrappers[oi].object=null;
- Object o = new Object();
- map.put(o,wrappers[oi]);
- e.object = o;
- e = e.next;
- }
- }
-
- public Object put(Object key, Object value) {
- ObjectWrapper o = (ObjectWrapper) map.get(key);
- if (o != null) {
- /*
- * this should never be the case, we only do a put on a cache miss which
- * means the current value wasn't in the cache. However if the user screws
- * up or wants to use this as a fixed size hash and puts the same thing in
- * the list twice then we update the value and more the key to the front of the
- * most recently used list.
- */
-
- // Move o's partner in the list to front
- DoubleLinkedListElement e = o.listItem;
-
- //move to front
- if (e != first) {
- //remove list item
- e.prev.next = e.next;
- if (e.next != null) {
- e.next.prev = e.prev;
- }
- else { //were moving last
- last = e.prev;
- }
-
- //put list item in front
- e.next = first;
- first.prev = e;
- e.prev = null;
-
- //update first
- first = e;
- }
- return o.object;
- }
- // Put o in the front and remove the last one
- lastKey = last.object; // store key to remove from hash later
- last.object = key; //update list element with new key
-
- // connect list item to front of list
- last.next = first;
- first.prev = last;
-
- // update first and last value
- first = last;
- last = last.prev;
- first.prev = null;
- last.next = null;
-
- // remove old value from cache
- temp = (ObjectWrapper) map.remove(lastKey);
- //update wrapper
- temp.object = value;
- temp.listItem = first;
-
- map.put(key, temp);
- return null;
- }
-
- public Object get(Object key) {
- ObjectWrapper o = (ObjectWrapper) map.get(key);
- if (o != null) {
- // Move it to the front
- DoubleLinkedListElement e = o.listItem;
-
- //move to front
- if (e != first) {
- //remove list item
- e.prev.next = e.next;
- if (e.next != null) {
- e.next.prev = e.prev;
- }
- else { //were moving last
- last = e.prev;
- }
- //put list item in front
- e.next = first;
- first.prev = e;
- e.prev = null;
-
- //update first
- first = e;
- }
- return o.object;
- }
- else {
- return null;
- }
- }
-
-
- public boolean containsKey(Object key) {
- return map.containsKey(key);
- }
-
- public boolean containsValue(Object value) {
- return map.containsValue(value);
- }
-
- public Set entrySet() {
- return map.entrySet();
- }
-
- public boolean isEmpty() {
- return map.isEmpty();
- }
-
- public Set keySet() {
- return map.keySet();
- }
-
- public void putAll(Map t) {
- map.putAll(t);
- }
-
- public Object remove(Object key) {
- return map.remove(key);
- }
-
- public int size() {
- return map.size();
- }
-
- public Collection values() {
- return map.values();
- }
-}
-
-class ObjectWrapper {
-
- public Object object;
- public DoubleLinkedListElement listItem;
-
- public ObjectWrapper(Object o,DoubleLinkedListElement li) {
- object = o;
- listItem = li;
- }
-
- public Object getObject() {
- return object;
- }
-
- public DoubleLinkedListElement getListItem() {
- return listItem;
- }
-
- public void setObject(Object o) {
- object = o;
- }
-
- public void setListItem(DoubleLinkedListElement li) {
- listItem = li;
- }
-
- public boolean eqauls(Object o) {
- return object.equals(o);
- }
-}
-
-class DoubleLinkedListElement {
-
- public DoubleLinkedListElement prev;
- public DoubleLinkedListElement next;
- public Object object;
-
- public DoubleLinkedListElement(DoubleLinkedListElement p,
- DoubleLinkedListElement n,
- Object o) {
- prev = p;
- next = n;
- object = o;
-
- if (p != null) {
- p.next = this;
- }
-
- if (n != null) {
- n.prev = this;
- }
- }
-}
-
-class DoubleLinkedList {
-
- DoubleLinkedListElement first;
- DoubleLinkedListElement last;
- DoubleLinkedListElement current;
-
- public DoubleLinkedList() {
- first = null;
- last = null;
- current = null;
- }
-
- public void addFirst(Object o) {
- first = new DoubleLinkedListElement(null, first, o);
-
- if (current.next == null) {
- last = current;
- }
- }
-
- public void addLast(Object o) {
- last = new DoubleLinkedListElement(last, null, o);
-
- if (current.prev == null) {
- first = current;
- }
- }
-
- public void insert(Object o) {
- if (current == null) {
- current = new DoubleLinkedListElement(null, null, o);
- }
- else {
- current = new DoubleLinkedListElement(current.prev, current, o);
- }
-
- if (current.prev == null) {
- first = current;
- }
-
- if (current.next == null) {
- last = current;
- }
- }
-
- public DoubleLinkedListElement getFirst() {
- current = first;
- return first;
- }
-
- public DoubleLinkedListElement getLast() {
- current = last;
- return last;
- }
-
- public DoubleLinkedListElement getCurrent() {
- return current;
- }
-
- public DoubleLinkedListElement next() {
- if (current.next != null) {
- current = current.next;
- }
- return current;
- }
-
- public DoubleLinkedListElement prev() {
- if (current.prev != null) {
- current = current.prev;
- }
- return current;
+ public Cache(final int capacity) {
+ this.capacity = capacity;
}
@Override
- public String toString() {
- DoubleLinkedListElement e = first;
- String s = "[" + e.object.toString();
-
- e = e.next;
-
- while (e != null) {
- s = s + ", " + e.object.toString();
- e = e.next;
- }
-
- s = s + "]";
-
- return s;
+ protected boolean removeEldestEntry(Map.Entry<K,V> eldest) {
+ return this.size() > this.capacity;
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4da7f4c6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CachedFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CachedFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CachedFeatureGenerator.java
index 2bdec5b..afb0a2c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CachedFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CachedFeatureGenerator.java
@@ -32,14 +32,14 @@ public class CachedFeatureGenerator implements AdaptiveFeatureGenerator {
private String[] prevTokens;
- private Cache contextsCache;
+ private Cache<Integer, List<String>> contextsCache;
private long numberOfCacheHits;
private long numberOfCacheMisses;
public CachedFeatureGenerator(AdaptiveFeatureGenerator... generators) {
this.generator = new AggregatedFeatureGenerator(generators);
- contextsCache = new Cache(100);
+ contextsCache = new Cache<>(100);
}
@SuppressWarnings("unchecked")
@@ -49,7 +49,7 @@ public class CachedFeatureGenerator implements AdaptiveFeatureGenerator {
List<String> cacheFeatures;
if (tokens == prevTokens) {
- cacheFeatures = (List<String>) contextsCache.get(index);
+ cacheFeatures = contextsCache.get(index);
if (cacheFeatures != null) {
numberOfCacheHits++;
@@ -62,7 +62,7 @@ public class CachedFeatureGenerator implements AdaptiveFeatureGenerator {
prevTokens = tokens;
}
- cacheFeatures = new ArrayList<String>();
+ cacheFeatures = new ArrayList<>();
numberOfCacheMisses++;
[47/50] [abbrv] opennlp git commit: OPENNLP-895: Include brat and
morfologik modules in distribution
Posted by jo...@apache.org.
OPENNLP-895: Include brat and morfologik modules in distribution
This closes #18
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/22108687
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/22108687
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/22108687
Branch: refs/heads/889
Commit: 221086874fcdaf60798df9c3aebbcbf0c4433865
Parents: 9cb610b
Author: smarthi <sm...@apache.org>
Authored: Fri Dec 23 11:42:21 2016 -0500
Committer: Kottmann <jo...@apache.org>
Committed: Sat Dec 24 17:17:11 2016 +0100
----------------------------------------------------------------------
NOTICE | 2 +-
opennlp-distr/src/main/assembly/bin.xml | 18 +-
opennlp-distr/src/main/readme/LICENSE | 347 +++++++++++++++++++++++++++
opennlp-distr/src/main/readme/NOTICE | 2 +-
4 files changed, 366 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/22108687/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
index a86854d..c0b8394 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache OpenNLP
-Copyright 2010, 2015 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
http://git-wip-us.apache.org/repos/asf/opennlp/blob/22108687/opennlp-distr/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml
index 926a9eb..5ec7ad7 100644
--- a/opennlp-distr/src/main/assembly/bin.xml
+++ b/opennlp-distr/src/main/assembly/bin.xml
@@ -34,6 +34,8 @@
<includes>
<include>org.apache.opennlp:opennlp-tools</include>
<include>org.apache.opennlp:opennlp-uima</include>
+ <include>org.apache.opennlp:opennlp-brat-annotator</include>
+ <include>org.apache.opennlp:opennlp-morfologik-addon</include>
</includes>
<unpack>false</unpack>
<useProjectArtifact>false</useProjectArtifact>
@@ -90,7 +92,21 @@
<directoryMode>755</directoryMode>
<outputDirectory>docs/apidocs/opennlp-tools</outputDirectory>
</fileSet>
-
+
+ <fileSet>
+ <directory>../opennlp-brat-annotator/target/apidocs</directory>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ <outputDirectory>docs/apidocs/opennlp-brat-annotator</outputDirectory>
+ </fileSet>
+
+ <fileSet>
+ <directory>../opennlp-morfologik-addon/target/apidocs</directory>
+ <fileMode>644</fileMode>
+ <directoryMode>755</directoryMode>
+ <outputDirectory>docs/apidocs/opennlp-morfologik-addon</outputDirectory>
+ </fileSet>
+
<fileSet>
<directory>../opennlp-uima/target/apidocs</directory>
<fileMode>644</fileMode>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/22108687/opennlp-distr/src/main/readme/LICENSE
----------------------------------------------------------------------
diff --git a/opennlp-distr/src/main/readme/LICENSE b/opennlp-distr/src/main/readme/LICENSE
index 576b4cf..842bd63 100644
--- a/opennlp-distr/src/main/readme/LICENSE
+++ b/opennlp-distr/src/main/readme/LICENSE
@@ -228,3 +228,350 @@ The following license applies to the Snowball stemmers:
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+
+
+The CDDL 1.1 license applies to jersey and some of its dependencies:
+
+COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1
+
+1. Definitions.
+
+ 1.1. \u201cContributor\u201d means each individual or entity that creates or contributes
+ to the creation of Modifications.
+
+ 1.2. \u201cContributor Version\u201d means the combination of the Original Software, prior
+ Modifications used by a Contributor (if any), and the Modifications made by that
+ particular Contributor.
+
+ 1.3. \u201cCovered Software\u201d means (a) the Original Software, or (b) Modifications
+ or (c) the combination of files containing Original Software with
+ files containing Modifications, in each case including portions thereof.
+
+ 1.4. \u201cExecutable\u201d means the Covered Software in any form other than Source Code.
+
+ 1.5. \u201cInitial Developer\u201d means the individual or entity that first makes
+ Original Software available under this License.
+
+ 1.6. \u201cLarger Work\u201d means a work which combines Covered Software or portions
+ thereof with code not governed by the terms of this License.
+
+ 1.7. \u201cLicense\u201d means this document.
+
+ 1.8. \u201cLicensable\u201d means having the right to grant, to the maximum extent possible,
+ whether at the time of the initial grant or subsequently acquired, any and
+ all of the rights conveyed herein.
+
+ 1.9. \u201cModifications\u201d means the Source Code and Executable form of any of the following:
+
+ A. Any file that results from an addition to, deletion from or modification of the
+ contents of a file containing Original Software or previous Modifications;
+
+ B. Any new file that contains any part of the Original Software or previous
+ Modification; or
+
+ C. Any new file that is contributed or otherwise made available under the terms
+ of this License.
+
+ 1.10. \u201cOriginal Software\u201d means the Source Code and Executable form of computer
+ software code that is originally released under this License.
+
+ 1.11. \u201cPatent Claims\u201d means any patent claim(s), now owned or hereafter acquired,
+ including without limitation, method, process, and apparatus claims, in any patent
+ Licensable by grantor.
+
+ 1.12. \u201cSource Code\u201d means (a) the common form of computer software code in which
+ modifications are made and (b) associated documentation included in or
+ with such code.
+
+ 1.13. \u201cYou\u201d (or \u201cYour\u201d) means an individual or a legal entity exercising rights
+ under, and complying with all of the terms of, this License. For legal
+ entities, \u201cYou\u201d includes any entity which controls, is controlled by, or
+ is under common control with You. For purposes of this definition, \u201ccontrol\u201d
+ means (a) the power, direct or indirect, to cause the direction or
+ management of such entity, whether by contract or otherwise, or (b) ownership
+ of more than fifty percent (50%) of the outstanding shares or beneficial
+ ownership of such entity.
+
+2. License Grants.
+
+ 2.1. The Initial Developer Grant.
+
+ Conditioned upon Your compliance with Section 3.1 below and subject to third party
+ intellectual property claims, the Initial Developer hereby grants You a world-wide,
+ royalty-free, non-exclusive license:
+
+ (a) under intellectual property rights (other than patent or trademark) Licensable
+ by Initial Developer, to use, reproduce, modify, display, perform, sublicense and
+ distribute the Original Software (or portions thereof), with or without Modifications,
+ and/or as part of a Larger Work; and
+
+ (b) under Patent Claims infringed by the making, using or selling of
+ Original Software, to make, have made, use, practice, sell, and offer for
+ sale, and/or otherwise dispose of the Original Software (or portions thereof).
+
+ (c) The licenses granted in Sections 2.1(a) and (b) are effective on the
+ date Initial Developer first distributes or otherwise makes the Original
+ Software available to a third party under the terms of this License.
+
+ (d) Notwithstanding Section 2.1(b) above, no patent license is granted:
+ (1) for code that You delete from the Original Software, or (2) for
+ infringements caused by: (i) the modification of the Original Software,
+ or (ii) the combination of the Original Software with other software or
+ devices.
+
+ 2.2. Contributor Grant.
+
+ Conditioned upon Your compliance with Section 3.1 below and subject to
+ third party intellectual property claims, each Contributor hereby grants
+ You a world-wide, royalty-free, non-exclusive license:
+
+ (a) under intellectual property rights (other than patent or trademark)
+ Licensable by Contributor to use, reproduce, modify, display, perform,
+ sublicense and distribute the Modifications created by such Contributor
+ (or portions thereof), either on an unmodified basis, with other Modifications,
+ as Covered Software and/or as part of a Larger Work; and
+
+ (b) under Patent Claims infringed by the making, using, or selling of Modifications
+ made by that Contributor either alone and/or in combination with its Contributor
+ Version (or portions of such combination), to make, use, sell, offer for sale,
+ have made, and/or otherwise dispose of: (1) Modifications made by that Contributor
+ (or portions thereof); and (2) the combination of Modifications made by that
+ Contributor with its Contributor Version (or portions of such combination).
+
+ (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the
+ date Contributor first distributes or otherwise makes the Modifications
+ available to a third party.
+
+ (d) Notwithstanding Section 2.2(b) above, no patent license is granted:
+ (1) for any code that Contributor has deleted from the Contributor Version;
+ (2) for infringements caused by: (i) third party modifications of Contributor
+ Version, or (ii) the combination of Modifications made by that Contributor with
+ other software (except as part of the Contributor Version) or other devices; or (3)
+ under Patent Claims infringed by Covered Software in the absence of Modifications
+ made by that Contributor.
+
+3. Distribution Obligations.
+
+ 3.1. Availability of Source Code.
+
+ Any Covered Software that You distribute or otherwise make available in Executable
+ form must also be made available in Source Code form and that Source Code form must
+ be distributed only under the terms of this License. You must include a copy of this
+ License with every copy of the Source Code form of the Covered Software You distribute
+ or otherwise make available. You must inform recipients of any such Covered Software in
+ Executable form as to how they can obtain such Covered Software in Source Code form in a
+ reasonable manner on or through a medium customarily used for software exchange.
+
+ 3.2. Modifications.
+
+ The Modifications that You create or to which You contribute are governed by the terms of
+ this License. You represent that You believe Your Modifications are Your original
+ creation(s) and/or You have sufficient rights to grant the rights conveyed by this License.
+
+ 3.3. Required Notices.
+
+ You must include a notice in each of Your Modifications that identifies
+ You as the Contributor of the Modification. You may not remove or alter
+ any copyright, patent or trademark notices contained within the Covered
+ Software, or any notices of licensing or any descriptive text giving
+ attribution to any Contributor or the Initial Developer.
+
+ 3.4. Application of Additional Terms.
+
+ You may not offer or impose any terms on any Covered Software in Source Code
+ form that alters or restricts the applicable version of this License or the
+ recipients' rights hereunder. You may choose to offer, and to charge a fee for,
+ warranty, support, indemnity or liability obligations to one or more recipients
+ of Covered Software. However, you may do so only on Your own behalf, and not on
+ behalf of the Initial Developer or any Contributor. You must make it absolutely
+ clear that any such warranty, support, indemnity or liability obligation is
+ offered by You alone, and You hereby agree to indemnify the Initial Developer
+ and every Contributor for any liability incurred by the Initial Developer or
+ such Contributor as a result of warranty, support, indemnity or liability
+ terms You offer.
+
+ 3.5. Distribution of Executable Versions.
+
+ You may distribute the Executable form of the Covered Software under
+ the terms of this License or under the terms of a license of Your choice,
+ which may contain terms different from this License, provided that You are in
+ compliance with the terms of this License and that the license for the
+ Executable form does not attempt to limit or alter the recipient's rights in
+ the Source Code form from the rights set forth in this License. If You
+ distribute the Covered Software in Executable form under a different license,
+ You must make it absolutely clear that any terms which differ from this License
+ are offered by You alone, not by the Initial Developer or Contributor. You
+ hereby agree to indemnify the Initial Developer and every Contributor for any
+ liability incurred by the Initial Developer or such Contributor as a result
+ of any such terms You offer.
+
+ 3.6. Larger Works.
+
+ You may create a Larger Work by combining Covered Software with other
+ code not governed by the terms of this License and distribute the Larger
+ Work as a single product. In such a case, You must make sure the requirements
+ of this License are fulfilled for the Covered Software.
+
+4. Versions of the License.
+
+ 4.1. New Versions.
+
+ Oracle is the initial license steward and may publish revised and/or new
+ versions of this License from time to time. Each version will be given a
+ distinguishing version number. Except as provided in Section 4.3, no one other
+ than the license steward has the right to modify this License.
+
+ 4.2. Effect of New Versions.
+
+ You may always continue to use, distribute or otherwise make the Covered
+ Software available under the terms of the version of the License under
+ which You originally received the Covered Software. If the Initial Developer
+ includes a notice in the Original Software prohibiting it from being distributed
+ or otherwise made available under any subsequent version of the License, You
+ must distribute and make the Covered Software available under the terms of
+ the version of the License under which You originally received the Covered
+ Software. Otherwise, You may also choose to use, distribute or otherwise
+ make the Covered Software available under the terms of any subsequent
+ version of the License published by the license steward.
+
+ 4.3. Modified Versions.
+
+ When You are an Initial Developer and You want to create a new license for
+ Your Original Software, You may create and use a modified version of
+ this License if You: (a) rename the license and remove any references to the
+ name of the license steward (except to note that the license differs from
+ this License); and (b) otherwise make it clear that the license contains
+ terms which differ from this License.
+
+5. DISCLAIMER OF WARRANTY.
+
+ COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN \u201cAS IS\u201d BASIS, WITHOUT
+ WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION,
+ WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE,
+ FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE
+ QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED
+ SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY
+ OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR
+ CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE.
+ NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
+
+6. TERMINATION.
+
+ 6.1. This License and the rights granted hereunder will terminate automatically
+ if You fail to comply with terms herein and fail to cure such breach within 30
+ days of becoming aware of the breach. Provisions which, by their nature, must
+ remain in effect beyond the termination of this License shall survive.
+
+ 6.2. If You assert a patent infringement claim (excluding declaratory judgment
+ actions) against Initial Developer or a Contributor (the Initial Developer or
+ Contributor against whom You assert such claim is referred to as \u201cParticipant\u201d)
+ alleging that the Participant Software (meaning the Contributor Version where
+ the Participant is a Contributor or the Original Software where the Participant
+ is the Initial Developer) directly or indirectly infringes any patent, then any
+ and all rights granted directly or indirectly to You by such Participant, the
+ Initial Developer (if the Initial Developer is not the Participant) and all Contributors
+ under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from
+ Participant terminate prospectively and automatically at the expiration of such
+ 60 day notice period, unless if within such 60 day period You withdraw Your claim
+ with respect to the Participant Software against such Participant either unilaterally
+ or pursuant to a written agreement with Participant.
+
+ 6.3. If You assert a patent infringement claim against Participant alleging that the
+ Participant Software directly or indirectly infringes any patent where such claim
+ is resolved (such as by license or settlement) prior to the initiation of patent
+ infringement litigation, then the reasonable value of the licenses granted by
+ such Participant under Sections 2.1 or 2.2 shall be taken into account in determining
+ the amount or value of any payment or license.
+
+ 6.4. In the event of termination under Sections 6.1 or 6.2 above, all end user
+ licenses that have been validly granted by You or any distributor hereunder prior
+ to termination (excluding licenses granted to You by any distributor) shall
+ survive termination.
+
+7. LIMITATION OF LIABILITY.
+
+ UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE),
+ CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR,
+ OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE
+ LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK
+ STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES
+ OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH
+ DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR
+ PERSONAL INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE
+ LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR
+ LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION
+ MAY NOT APPLY TO YOU.
+
+8. U.S. GOVERNMENT END USERS.
+
+ The Covered Software is a \u201ccommercial item,\u201d as that term is defined in 48
+ C.F.R. 2.101 (Oct. 1995), consisting of \u201ccommercial computer software\u201d (as that term
+ is defined at 48 C.F.R. � 252.227-7014(a)(1)) and \u201ccommercial computer software
+ documentation\u201d as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with
+ 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S.
+ Government End Users acquire Covered Software with only those rights set forth herein.
+ This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR,
+ or other clause or provision that addresses Government rights in computer software
+ under this License.
+
+9. MISCELLANEOUS.
+
+ This License represents the complete agreement concerning subject matter hereof.
+ If any provision of this License is held to be unenforceable, such provision
+ shall be reformed only to the extent necessary to make it enforceable.
+ This License shall be governed by the law of the jurisdiction specified in a
+ notice contained within the Original Software (except to the extent applicable law,
+ if any, provides otherwise), excluding such jurisdiction's conflict-of-law provisions.
+ Any litigation relating to this License shall be subject to the jurisdiction of the
+ courts located in the jurisdiction and venue specified in a notice contained within
+ the Original Software, with the losing party responsible for costs, including, without
+ limitation, court costs and reasonable attorneys' fees and expenses. The application
+ of the United Nations Convention on Contracts for the International Sale of Goods is
+ expressly excluded. Any law or regulation which provides that the language of a contract
+ shall be construed against the drafter shall not apply to this License. You agree that
+ You alone are responsible for compliance with the United States export administration
+ regulations (and the export control laws and regulation of any other countries) when
+ You use, distribute or otherwise make available any Covered Software.
+
+10. RESPONSIBILITY FOR CLAIMS.
+
+ As between Initial Developer and the Contributors, each party is responsible for claims
+ and damages arising, directly or indirectly, out of its utilization of rights under
+ this License and You agree to work with Initial Developer and Contributors to distribute
+ such responsibility on an equitable basis. Nothing herein is intended or shall be
+ deemed to constitute any admission of liability.
+
+NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)
+
+The code released under the CDDL shall be governed by the laws of the State
+of California (excluding conflict-of-law provisions). Any litigation relating
+to this License shall be subject to the jurisdiction of the Federal Courts of
+the Northern District of California and the state courts of the State of
+California, with venue lying in Santa Clara County, California.
+
+
+The BSD 2-clause license applies to the morfologik dependencies:
+Copyright (c) <YEAR> <OWNER>
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/opennlp/blob/22108687/opennlp-distr/src/main/readme/NOTICE
----------------------------------------------------------------------
diff --git a/opennlp-distr/src/main/readme/NOTICE b/opennlp-distr/src/main/readme/NOTICE
index 73fb1d7..fa391c4 100644
--- a/opennlp-distr/src/main/readme/NOTICE
+++ b/opennlp-distr/src/main/readme/NOTICE
@@ -1,5 +1,5 @@
Apache OpenNLP
-Copyright 2010, 2013 The Apache Software Foundation
+Copyright 2017 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
[36/50] [abbrv] opennlp git commit: Improve error reporting on brat
parser crashes
Posted by jo...@apache.org.
Improve error reporting on brat parser crashes
Parsing errors are now catched and wrapped in
an IOException containing the document id for
tracking down the actual file.
See issue OPENNLP-881
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/0debe9c1
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/0debe9c1
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/0debe9c1
Branch: refs/heads/889
Commit: 0debe9c1fc70c31b491ac99c4373040177435ef3
Parents: 727964d
Author: J�rn Kottmann <jo...@apache.org>
Authored: Thu Nov 10 01:07:14 2016 +0100
Committer: Joern Kottmann <jo...@apache.org>
Committed: Wed Dec 21 10:54:53 2016 +0100
----------------------------------------------------------------------
.../formats/brat/BratAnnotationStream.java | 25 +++++++++++++-------
1 file changed, 16 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/0debe9c1/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
index 91a2916..125e662 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
@@ -82,9 +82,14 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
String coveredText = line.subSequence(values[firstTextTokenIndex].getStart(),
values[values.length - 1].getEnd()).toString();
- return new SpanAnnotation(id, type,
- new Span(parseInt(values[BEGIN_OFFSET]
- .getCoveredText(line).toString()), endOffset, type), coveredText);
+ try {
+ return new SpanAnnotation(id, type,
+ new Span(parseInt(values[BEGIN_OFFSET]
+ .getCoveredText(line).toString()), endOffset, type), coveredText);
+ }
+ catch (IllegalArgumentException e) {
+ throw new InvalidFormatException(e);
+ }
}
else {
throw new InvalidFormatException("Line must have at least 5 fields");
@@ -172,18 +177,20 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
BratAnnotationParser parser = parsers.get(typeClass);
if (parser == null) {
- throw new IOException("Failed to parse ann document with id " + id +
+ throw new IOException("Failed to parse ann document with id " + id + ".ann" +
" type class, no parser registered: " + tokens[BratAnnotationParser.TYPE_OFFSET]
.getCoveredText(line).toString());
}
- return parser.parse(tokens, line);
+ try {
+ return parser.parse(tokens, line);
+ }
+ catch (IOException e) {
+ throw new IOException(String.format("Failed to parse ann document with id [%s.ann]", id), e);
+ }
}
}
- else {
- return null;
- }
-
+
return null;
}
[29/50] [abbrv] opennlp git commit: OPENNLP-871: Cleanup code base
for release
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
index 2c31345..380350e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
@@ -17,7 +17,6 @@
package opennlp.tools.doccat;
import java.io.IOException;
-import java.io.ObjectStreamException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -25,7 +24,6 @@ import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-
import opennlp.tools.ml.EventTrainer;
import opennlp.tools.ml.TrainerFactory;
import opennlp.tools.ml.model.MaxentModel;
@@ -33,7 +31,6 @@ import opennlp.tools.tokenize.SimpleTokenizer;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
-import opennlp.tools.util.model.ModelUtil;
/**
* Maxent implementation of {@link DocumentCategorizer}.
@@ -119,7 +116,7 @@ public class DocumentCategorizerME implements DocumentCategorizer {
* @return the score map
*/
public Map<String, Double> scoreMap(String text) {
- Map<String, Double> probDist = new HashMap<String, Double>();
+ Map<String, Double> probDist = new HashMap<>();
double[] categorize = categorize(text);
int catSize = getNumberOfCategories();
@@ -139,7 +136,7 @@ public class DocumentCategorizerME implements DocumentCategorizer {
* @return the sorted score map
*/
public SortedMap<Double, Set<String>> sortedScoreMap(String text) {
- SortedMap<Double, Set<String>> descendingMap = new TreeMap<Double, Set<String>>();
+ SortedMap<Double, Set<String>> descendingMap = new TreeMap<>();
double[] categorize = categorize(text);
int catSize = getNumberOfCategories();
for (int i = 0; i < catSize; i++) {
@@ -180,7 +177,7 @@ public class DocumentCategorizerME implements DocumentCategorizer {
TrainingParameters mlParams, DoccatFactory factory)
throws IOException {
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
EventTrainer trainer = TrainerFactory.getEventTrainer(
mlParams.getSettings(), manifestInfoEntries);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentSample.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentSample.java
index c6c1852..47828ab 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentSample.java
@@ -50,7 +50,7 @@ public class DocumentSample {
}
this.category = category;
- this.text = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(text)));
+ this.text = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(text)));
if(extraInformation == null) {
this.extraInformation = Collections.emptyMap();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/NGramFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/NGramFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/NGramFeatureGenerator.java
index 49e1736..ef5d2a3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/NGramFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/NGramFeatureGenerator.java
@@ -69,7 +69,7 @@ public class NGramFeatureGenerator implements FeatureGenerator {
*/
public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInfo) {
- List<String> features = new ArrayList<String>();
+ List<String> features = new ArrayList<>();
for (int i = 0; i <= text.length - minGram; i++) {
String feature = "ng=";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseLink.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseLink.java b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseLink.java
index 089d9b7..6e06beb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseLink.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/BaseLink.java
@@ -29,7 +29,7 @@ public abstract class BaseLink {
private String itemID;
private String itemName;
private String itemType;
- private HashMap<String, Double> scoreMap = new HashMap<String, Double>();
+ private HashMap<String, Double> scoreMap = new HashMap<>();
public BaseLink() {
}
@@ -150,12 +150,6 @@ public abstract class BaseLink {
if (!Objects.equals(this.itemID, other.itemID)) {
return false;
}
- if (!Objects.equals(this.itemName, other.itemName)) {
- return false;
- }
- if (!Objects.equals(this.itemType, other.itemType)) {
- return false;
- }
- return true;
+ return Objects.equals(this.itemName, other.itemName) && Objects.equals(this.itemType, other.itemType);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedSpan.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedSpan.java b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedSpan.java
index ff4757f..c44aec8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedSpan.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/LinkedSpan.java
@@ -136,12 +136,6 @@ public class LinkedSpan<T extends BaseLink> extends Span {
if (!Objects.equals(this.linkedEntries, other.linkedEntries)) {
return false;
}
- if (this.sentenceid != other.sentenceid) {
- return false;
- }
- if (!Objects.equals(this.searchTerm, other.searchTerm)) {
- return false;
- }
- return true;
+ return this.sentenceid == other.sentenceid && Objects.equals(this.searchTerm, other.searchTerm);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
index 1d8d4e9..6825d80 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
@@ -82,8 +82,8 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
public NameSample read() throws IOException {
- List<String> sentence = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
+ List<String> sentence = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
boolean isClearAdaptiveData = false;
@@ -116,7 +116,7 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
if (sentence.size() > 0) {
// convert name tags into spans
- List<Span> names = new ArrayList<Span>();
+ List<Span> names = new ArrayList<>();
int beginIndex = -1;
int endIndex = -1;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
index 8d2df4b..7cd3810 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
@@ -102,7 +102,7 @@ public class Conll02NameSampleStream implements ObjectStream<NameSample>{
this.types = types;
}
- static final Span extract(int begin, int end, String beginTag) throws InvalidFormatException {
+ static Span extract(int begin, int end, String beginTag) throws InvalidFormatException {
String type = beginTag.substring(2);
@@ -128,8 +128,8 @@ public class Conll02NameSampleStream implements ObjectStream<NameSample>{
public NameSample read() throws IOException {
- List<String> sentence = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
+ List<String> sentence = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
boolean isClearAdaptiveData = false;
@@ -162,7 +162,7 @@ public class Conll02NameSampleStream implements ObjectStream<NameSample>{
if (sentence.size() > 0) {
// convert name tags into spans
- List<Span> names = new ArrayList<Span>();
+ List<Span> names = new ArrayList<>();
int beginIndex = -1;
int endIndex = -1;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
index 78c0ee8..07b62e8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
@@ -15,15 +15,12 @@
package opennlp.tools.formats;
-import static opennlp.tools.formats.Conll02NameSampleStream.extract;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
-
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
@@ -31,6 +28,8 @@ import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.StringUtil;
+import static opennlp.tools.formats.Conll02NameSampleStream.extract;
+
/**
* An import stream which can parse the CONLL03 data.
*/
@@ -93,8 +92,8 @@ public class Conll03NameSampleStream implements ObjectStream<NameSample>{
public NameSample read() throws IOException {
- List<String> sentence = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
+ List<String> sentence = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
boolean isClearAdaptiveData = false;
@@ -133,7 +132,7 @@ public class Conll03NameSampleStream implements ObjectStream<NameSample>{
if (sentence.size() > 0) {
// convert name tags into spans
- List<Span> names = new ArrayList<Span>();
+ List<Span> names = new ArrayList<>();
int beginIndex = -1;
int endIndex = -1;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStream.java
index 82ac5eb..b3b88fc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ConllXPOSSampleStream.java
@@ -68,8 +68,8 @@ public class ConllXPOSSampleStream extends FilterObjectStream<String, POSSample>
// paragraph get lines
BufferedReader reader = new BufferedReader(new StringReader(paragraph));
- List<String> tokens = new ArrayList<String>(100);
- List<String> tags = new ArrayList<String>(100);
+ List<String> tokens = new ArrayList<>(100);
+ List<String> tags = new ArrayList<>(100);
String line;
while ((line = reader.readLine()) != null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
index 10c960f..d176f8b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
@@ -126,9 +126,9 @@ public class ADChunkSampleStream implements ObjectStream<ChunkSample> {
// skip this one
} else {
Node root = paragraph.getRoot();
- List<String> sentence = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
- List<String> target = new ArrayList<String>();
+ List<String> sentence = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
+ List<String> target = new ArrayList<>();
processRoot(root, sentence, tags, target);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
index 5e131f2..abf6dab 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
@@ -80,7 +80,7 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
private static final Map<String, String> HAREM;
static {
- Map<String, String> harem = new HashMap<String, String>();
+ Map<String, String> harem = new HashMap<>();
final String person = "person";
harem.put("hum", person);
@@ -226,7 +226,7 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
}
}
- int textID = -1;
+ private int textID = -1;
public NameSample read() throws IOException {
@@ -242,8 +242,8 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
}
Node root = paragraph.getRoot();
- List<String> sentence = new ArrayList<String>();
- List<Span> names = new ArrayList<Span>();
+ List<String> sentence = new ArrayList<>();
+ List<Span> names = new ArrayList<>();
process(root, sentence, names);
return new NameSample(sentence.toArray(new String[sentence.size()]),
@@ -359,17 +359,12 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
} else {
error = true;
}
- if (error) {
-// Maybe it is not the same NER, skip it.
-// System.err.println("Missing NER start for sentence [" + sentence
-// + "] node [" + leaf + "]");
- }
}
}
private List<String> processLexeme(String lexemeStr) {
- List<String> out = new ArrayList<String>();
+ List<String> out = new ArrayList<>();
String[] parts = underlinePattern.split(lexemeStr);
for (String tok : parts) {
if(tok.length() > 1 && !alphanumericPattern.matcher(tok).matches()) {
@@ -384,8 +379,8 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
private List<String> processTok(String tok) {
boolean tokAdded = false;
String original = tok;
- List<String> out = new ArrayList<String>();
- LinkedList<String> suffix = new LinkedList<String>();
+ List<String> out = new ArrayList<>();
+ LinkedList<String> suffix = new LinkedList<>();
char first = tok.charAt(0);
if (first == '�') {
out.add(Character.toString(first));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
index ed030f2..ff57d83 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
@@ -124,8 +124,8 @@ public class ADPOSSampleStream implements ObjectStream<POSSample> {
Sentence paragraph;
while ((paragraph = this.adSentenceStream.read()) != null) {
Node root = paragraph.getRoot();
- List<String> sentence = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
+ List<String> sentence = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
process(root, sentence, tags);
return new POSSample(sentence, tags);
@@ -166,8 +166,8 @@ public class ADPOSSampleStream implements ObjectStream<POSSample> {
StringTokenizer tokenizer = new StringTokenizer(lexeme, "_");
if (tokenizer.countTokens() > 0) {
- List<String> toks = new ArrayList<String>(tokenizer.countTokens());
- List<String> tagsWithCont = new ArrayList<String>(
+ List<String> toks = new ArrayList<>(tokenizer.countTokens());
+ List<String> tagsWithCont = new ArrayList<>(
tokenizer.countTokens());
toks.add(tokenizer.nextToken());
tagsWithCont.add("B-" + tag);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
index dcae4e5..8045352 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStreamFactory.java
@@ -81,10 +81,8 @@ public class ADPOSSampleStreamFactory extends
CmdLineUtil.handleCreateObjectStreamError(ex);
}
- ADPOSSampleStream sentenceStream = new ADPOSSampleStream(lineStream,
+ return new ADPOSSampleStream(lineStream,
params.getExpandME(), params.getIncludeFeatures());
-
- return sentenceStream;
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
index ed4f58b..d3a04df 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
@@ -129,7 +129,7 @@ public class ADSentenceSampleStream implements ObjectStream<SentenceSample> {
}
StringBuilder document = new StringBuilder();
- List<Span> sentences = new ArrayList<Span>();
+ List<Span> sentences = new ArrayList<>();
do {
do {
if (!isTitle || (isTitle && isIncludeTitles)) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
index fa02a92..9e3cec7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStreamFactory.java
@@ -79,9 +79,6 @@ public class ADSentenceSampleStreamFactory extends
CmdLineUtil.handleCreateObjectStreamError(ex);
}
- ADSentenceSampleStream sentenceStream = new ADSentenceSampleStream(
- lineStream, includeTitle);
-
- return sentenceStream;
+ return new ADSentenceSampleStream(lineStream, includeTitle);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index 275cf4d..5497816 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -138,9 +138,6 @@ public class ADSentenceStream extends
if(isBox) boxTag = " box";
if(start > 0) {
meta = line.substring(0, start) + " p=" + para + titleTag + boxTag + metaFromSource;
- } else {
- // rare case were there is no space between id and the sentence.
- // will use previous meta for now
}
}
sentence.setText(text);
@@ -154,7 +151,7 @@ public class ADSentenceStream extends
}
// got the root. Add it to the stack
- Stack<Node> nodeStack = new Stack<Node>();
+ Stack<Node> nodeStack = new Stack<>();
root.setSyntacticTag("ROOT");
root.setLevel(0);
@@ -381,7 +378,7 @@ public class ADSentenceStream extends
/** Represents the AD node */
public class Node extends TreeElement {
- private List<TreeElement> elems = new ArrayList<TreeElement>();
+ private List<TreeElement> elems = new ArrayList<>();
public void addElement(TreeElement element) {
elems.add(element);
@@ -546,12 +543,10 @@ public class ADSentenceStream extends
if(sentenceStarted) {
if (sentEnd.matcher(line).matches() || extEnd.matcher(line).matches()) {
sentenceStarted = false;
- } else if(line.startsWith("A1")) {
- // skip
- } else {
+ } else if (!line.startsWith("A1")) {
sentence.append(line).append('\n');
}
- } else {
+ } else {
if (sentStart.matcher(line).matches()) {
sentenceStarted = true;
} else if(paraStart.matcher(line).matches()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
index 723ddbc..9ce5b3b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
@@ -41,8 +41,7 @@ public class AnnotationConfiguration {
public AnnotationConfiguration(Map<String, String> typeToClassMap) {
- this.typeToClassMap = Collections.unmodifiableMap(
- new HashMap<String, String>(typeToClassMap));
+ this.typeToClassMap = Collections.unmodifiableMap(new HashMap<>(typeToClassMap));
}
public String getTypeClass(String type) {
@@ -51,7 +50,7 @@ public class AnnotationConfiguration {
public static AnnotationConfiguration parse(InputStream in) throws IOException {
- Map<String, String> typeToClassMap = new HashMap<String, String>();
+ Map<String, String> typeToClassMap = new HashMap<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(in, Charset.forName("UTF-8")));
@@ -63,9 +62,7 @@ public class AnnotationConfiguration {
line = line.trim();
if (line.isEmpty()) {
- continue;
} else if (line.startsWith("#")) {
- continue;
} else if (line.startsWith("[") && line.endsWith("]")) {
sectionType = line.substring(line.indexOf('[') + 1, line.indexOf(']'));
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
index 9d4b0f2..91a2916 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratAnnotationStream.java
@@ -141,8 +141,7 @@ public class BratAnnotationStream implements ObjectStream<BratAnnotation> {
}
}
- private final Map<String, BratAnnotationParser> parsers =
- new HashMap<String, BratAnnotationParser>();
+ private final Map<String, BratAnnotationParser> parsers = new HashMap<>();
private final AnnotationConfiguration config;
private final BufferedReader reader;
private final String id;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocumentStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocumentStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocumentStream.java
index ba82089..4e702cc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocumentStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratDocumentStream.java
@@ -33,7 +33,7 @@ import opennlp.tools.util.ObjectStream;
public class BratDocumentStream implements ObjectStream<BratDocument> {
private AnnotationConfiguration config;
- private List<String> documentIds = new LinkedList<String>();
+ private List<String> documentIds = new LinkedList<>();
private Iterator<String> documentIdIterator;
/**
@@ -57,7 +57,7 @@ public class BratDocumentStream implements ObjectStream<BratDocument> {
this.config = config;
- Stack<File> directoryStack = new Stack<File>();
+ Stack<File> directoryStack = new Stack<>();
directoryStack.add(bratCorpusDirectory);
while (!directoryStack.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
index 5bb5744..6f75c3e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStream.java
@@ -67,7 +67,7 @@ public class BratNameSampleStream extends SegmenterObjectStream<BratDocument, Na
// to be able to print warning a set of entities id must be maintained
// to check if all entities have been used up after the matching is done
- Set<String> entityIdSet = new HashSet<String>();
+ Set<String> entityIdSet = new HashSet<>();
for (BratAnnotation ann : sample.getAnnotations()) {
if (ann instanceof SpanAnnotation) {
@@ -93,7 +93,7 @@ public class BratNameSampleStream extends SegmenterObjectStream<BratDocument, Na
// Currently we are missing all
- List<NameSample> samples = new ArrayList<NameSample>(sentences.length);
+ List<NameSample> samples = new ArrayList<>(sentences.length);
for (Span sentence : sentences) {
@@ -109,14 +109,14 @@ public class BratNameSampleStream extends SegmenterObjectStream<BratDocument, Na
// in the tokenIndexMap.
// The tokenIndexMap maps to the sentence local token index.
- Map<Integer, Integer> tokenIndexMap = new HashMap<Integer, Integer>();
+ Map<Integer, Integer> tokenIndexMap = new HashMap<>();
for (int i = 0; i < tokens.length; i++) {
tokenIndexMap.put(-(sentence.getStart() + tokens[i].getStart()), i);
tokenIndexMap.put(sentence.getStart() + tokens[i].getEnd(), i + 1);
}
- List<Span> names = new ArrayList<Span>();
+ List<Span> names = new ArrayList<>();
for (BratAnnotation ann : sample.getAnnotations()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/AbstractToSentenceSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/AbstractToSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/AbstractToSentenceSampleStream.java
index 5ef543a..96434e7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/AbstractToSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/AbstractToSentenceSampleStream.java
@@ -54,7 +54,7 @@ public abstract class AbstractToSentenceSampleStream<T> extends
protected abstract String[] toSentence(T sample);
public SentenceSample read() throws IOException {
- List<String[]> sentences = new ArrayList<String[]>();
+ List<String[]> sentences = new ArrayList<>();
T posSample;
int chunks = 0;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStream.java
index fd7dabd..ad049e0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToPOSSampleStream.java
@@ -41,8 +41,8 @@ public class ParseToPOSSampleStream extends FilterObjectStream<Parse, POSSample>
if (parse != null) {
- List<String> sentence = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
+ List<String> sentence = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
for(Parse tagNode : parse.getTagNodes()) {
sentence.add(tagNode.getCoveredText());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
index e455f01..4d6d083 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGenerator.java
@@ -41,7 +41,7 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener
protected static String[] getPrefixes(String lex) {
String[] prefs = new String[PREFIX_LENGTH];
- for (int li = 1, ll = PREFIX_LENGTH; li < ll; li++) {
+ for (int li = 1; li < PREFIX_LENGTH; li++) {
prefs[li] = lex.substring(0, Math.min(li + 1, lex.length()));
}
return prefs;
@@ -49,7 +49,7 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener
protected static String[] getSuffixes(String lex) {
String[] suffs = new String[SUFFIX_LENGTH];
- for (int li = 1, ll = SUFFIX_LENGTH; li < ll; li++) {
+ for (int li = 1; li < SUFFIX_LENGTH; li++) {
suffs[li] = lex.substring(Math.max(lex.length() - li - 1, 0));
}
return suffs;
@@ -78,7 +78,7 @@ public class DefaultLemmatizerContextGenerator implements LemmatizerContextGener
w0 = "w0=" + toks[index];
t0 = "t0=" + tags[index];
- List<String> features = new ArrayList<String>();
+ List<String> features = new ArrayList<>();
features.add(w0);
features.add(t0);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
index 0ae8e37..dde2d08 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
@@ -47,7 +47,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
* the input dictionary via inputstream
*/
public DictionaryLemmatizer(final InputStream dictionary) {
- this.dictMap = new HashMap<List<String>, String>();
+ this.dictMap = new HashMap<>();
final BufferedReader breader = new BufferedReader(new InputStreamReader(
dictionary));
String line;
@@ -80,13 +80,13 @@ public class DictionaryLemmatizer implements Lemmatizer {
* @return returns the dictionary keys
*/
private List<String> getDictKeys(final String word, final String postag) {
- final List<String> keys = new ArrayList<String>();
+ final List<String> keys = new ArrayList<>();
keys.addAll(Arrays.asList(word.toLowerCase(), postag));
return keys;
}
public String[] lemmatize(final String[] tokens, final String[] postags) {
- List<String> lemmas = new ArrayList<String>();
+ List<String> lemmas = new ArrayList<>();
for (int i = 0; i < tokens.length; i++) {
lemmas.add(this.apply(tokens[i], postags[i]));
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
index fe6dd69..a19adb4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
@@ -43,9 +43,9 @@ public LemmaSample(String[] tokens, String[] tags, String[] lemmas) {
validateArguments(tokens.length, tags.length, lemmas.length);
- this.tokens = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(tokens)));
- this.tags = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(tags)));
- this.lemmas = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(lemmas)));
+ this.tokens = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(tokens)));
+ this.tags = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(tags)));
+ this.lemmas = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(lemmas)));
}
/**
@@ -58,9 +58,9 @@ public LemmaSample(String[] tokens, String[] tags, String[] lemmas) {
validateArguments(tokens.size(), tags.size(), lemmas.size());
- this.tokens = Collections.unmodifiableList(new ArrayList<String>(tokens));
- this.tags = Collections.unmodifiableList(new ArrayList<String>(tags));
- this.lemmas = Collections.unmodifiableList(new ArrayList<String>(lemmas));
+ this.tokens = Collections.unmodifiableList(new ArrayList<>(tokens));
+ this.tags = Collections.unmodifiableList(new ArrayList<>(tags));
+ this.lemmas = Collections.unmodifiableList(new ArrayList<>(lemmas));
}
public String[] getTokens() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleEventStream.java
index 2a71be2..1a46f4a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleEventStream.java
@@ -45,7 +45,7 @@ public class LemmaSampleEventStream extends AbstractEventStream<LemmaSample> {
protected Iterator<Event> createEvents(LemmaSample sample) {
if (sample != null) {
- List<Event> events = new ArrayList<Event>();
+ List<Event> events = new ArrayList<>();
String[] toksArray = sample.getTokens();
String[] tagsArray = sample.getTags();
String[] lemmasArray = sample.getLemmas();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleSequenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleSequenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleSequenceStream.java
index 1cdfbcf..0940dc6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleSequenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleSequenceStream.java
@@ -52,7 +52,7 @@ public class LemmaSampleSequenceStream implements SequenceStream {
events[i] = new Event(tags[i], context);
}
- return new Sequence<LemmaSample>(events,sample);
+ return new Sequence<>(events,sample);
}
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleStream.java
index b59ea07..9a2a6f0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSampleStream.java
@@ -38,9 +38,9 @@ public class LemmaSampleStream extends FilterObjectStream<String, LemmaSample> {
public LemmaSample read() throws IOException {
- List<String> toks = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
- List<String> preds = new ArrayList<String>();
+ List<String> toks = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
+ List<String> preds = new ArrayList<>();
for (String line = samples.read(); line != null && !line.equals(""); line = samples.read()) {
String[] parts = line.split("\t");
@@ -55,8 +55,8 @@ public class LemmaSampleStream extends FilterObjectStream<String, LemmaSample> {
}
}
if (toks.size() > 0) {
- LemmaSample lemmaSample = new LemmaSample(toks.toArray(new String[toks.size()]), tags.toArray(new String[tags.size()]), preds.toArray(new String[preds.size()]));
- return lemmaSample;
+ return new LemmaSample(toks.toArray(new String[toks.size()]), tags.toArray(new String[tags.size()]),
+ preds.toArray(new String[preds.size()]));
}
else {
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/Lemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/Lemmatizer.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/Lemmatizer.java
index 2b6ab1e..d09a8a3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/Lemmatizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/Lemmatizer.java
@@ -29,6 +29,6 @@ public interface Lemmatizer {
*
* @return an array of lemma classes for each token in the sequence.
*/
- public String[] lemmatize(String[] toks, String tags[]);
+ String[] lemmatize(String[] toks, String tags[]);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerContextGenerator.java
index 5ea10c1..9513342 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerContextGenerator.java
@@ -31,6 +31,6 @@ public interface LemmatizerContextGenerator extends BeamSearchContextGenerator<S
* @param lemmas The previous decisions made in the tagging of this sequence. Only indices less than i will be examined.
* @return An array of predictive contexts on which a model basis its decisions.
*/
- public String[] getContext(int i, String[] toks, String[] tags, String[] lemmas);
+ String[] getContext(int i, String[] toks, String[] tags, String[] lemmas);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerFactory.java
index 605fc84..8415e57 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerFactory.java
@@ -37,9 +37,7 @@ public class LemmatizerFactory extends BaseToolFactory {
return new LemmatizerFactory();
}
try {
- LemmatizerFactory theFactory = ExtensionLoader.instantiateExtension(
- LemmatizerFactory.class, subclassName);
- return theFactory;
+ return ExtensionLoader.instantiateExtension(LemmatizerFactory.class, subclassName);
} catch (Exception e) {
String msg = "Could not instantiate the " + subclassName
+ ". The initialization throw an exception.";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerME.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerME.java
index f25e7c4..6d97b5f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmatizerME.java
@@ -79,7 +79,7 @@ public class LemmatizerME implements Lemmatizer {
this.model = model.getLemmatizerSequenceModel();
}
else {
- this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+ this.model = new opennlp.tools.ml.BeamSearch<>(beamSize,
(MaxentModel) model.getLemmatizerSequenceModel(), 0);
}
}
@@ -97,7 +97,7 @@ public String[] lemmatize(String[] toks, String[] tags) {
* @return the array of decoded lemmas
*/
public String[] decodeLemmas(String[] toks, String[] preds) {
- List<String> lemmas = new ArrayList<String>();
+ List<String> lemmas = new ArrayList<>();
for (int i = 0; i < toks.length; i++) {
String lemma = StringUtil.decodeShortestEditScript(toks[i].toLowerCase(), preds[i]);
//System.err.println("-> DEBUG: " + toks[i].toLowerCase() + " " + preds[i] + " " + lemma);
@@ -153,7 +153,7 @@ public String[] lemmatize(String[] toks, String[] tags) {
LemmatizerContextGenerator contextGenerator = posFactory.getContextGenerator();
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
TrainerType trainerType = TrainerFactory.getTrainerType(trainParams.getSettings());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/TrainerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/TrainerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/ml/TrainerFactory.java
index 19c9a79..207f830 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/TrainerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/TrainerFactory.java
@@ -42,7 +42,7 @@ public class TrainerFactory {
private static final Map<String, Class> BUILTIN_TRAINERS;
static {
- Map<String, Class> _trainers = new HashMap<String, Class>();
+ Map<String, Class> _trainers = new HashMap<>();
_trainers.put(GIS.MAXENT_VALUE, GIS.class);
_trainers.put(QNTrainer.MAXENT_QN_VALUE, QNTrainer.class);
_trainers.put(PerceptronTrainer.PERCEPTRON_VALUE, PerceptronTrainer.class);
@@ -89,21 +89,21 @@ public class TrainerFactory {
ExtensionLoader.instantiateExtension(EventTrainer.class, alogrithmValue);
return TrainerType.EVENT_MODEL_TRAINER;
}
- catch (ExtensionNotLoadedException e) {
+ catch (ExtensionNotLoadedException ignored) {
}
try {
ExtensionLoader.instantiateExtension(EventModelSequenceTrainer.class, alogrithmValue);
return TrainerType.EVENT_MODEL_SEQUENCE_TRAINER;
}
- catch (ExtensionNotLoadedException e) {
+ catch (ExtensionNotLoadedException ignored) {
}
try {
ExtensionLoader.instantiateExtension(SequenceTrainer.class, alogrithmValue);
return TrainerType.SEQUENCE_TRAINER;
}
- catch (ExtensionNotLoadedException e) {
+ catch (ExtensionNotLoadedException ignored) {
}
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/DataStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/DataStream.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/DataStream.java
index 769e138..f8a0268 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/DataStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/DataStream.java
@@ -32,13 +32,13 @@ public interface DataStream {
*
* @return the Object representing the data which is next in this DataStream
*/
- public Object nextToken();
+ Object nextToken();
/**
* Test whether there are any Events remaining in this EventStream.
*
* @return true if this DataStream has more data tokens
*/
- public boolean hasNext();
+ boolean hasNext();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GIS.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GIS.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GIS.java
index 194664e..7a57aa7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GIS.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GIS.java
@@ -63,11 +63,7 @@ public class GIS extends AbstractEventTrainer {
String algorithmName = getAlgorithm();
- if (algorithmName != null && !(MAXENT_VALUE.equals(algorithmName))) {
- return false;
- }
-
- return true;
+ return !(algorithmName != null && !(MAXENT_VALUE.equals(algorithmName)));
}
public boolean isSortAndMerge() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
index 97ff167..f2c3b82 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/RealBasicEventStream.java
@@ -27,7 +27,7 @@ import opennlp.tools.util.ObjectStream;
public class RealBasicEventStream implements ObjectStream<Event> {
ContextGenerator cg = new BasicContextGenerator();
- ObjectStream<String> ds;
+ private ObjectStream<String> ds;
public RealBasicEventStream(ObjectStream<String> ds) {
this.ds = ds;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/BinaryGISModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/BinaryGISModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/BinaryGISModelWriter.java
index 3faf337..5c3503d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/BinaryGISModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/BinaryGISModelWriter.java
@@ -31,7 +31,7 @@ import opennlp.tools.ml.model.AbstractModel;
* Model writer that saves models in binary format.
*/
public class BinaryGISModelWriter extends GISModelWriter {
- DataOutputStream output;
+ private DataOutputStream output;
/**
* Constructor which takes a GISModel and a File and prepares itself to write
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java
index a1120bf..45bc5c8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/GISModelWriter.java
@@ -123,7 +123,6 @@ public abstract class GISModelWriter extends AbstractModelWriter {
int[] predkeys = PARAMS[pid].getOutcomes();
// Arrays.sort(predkeys);
int numActive = predkeys.length;
- int[] activeOutcomes = predkeys;
double[] activeParams = PARAMS[pid].getParameters();
numParams += numActive;
@@ -135,7 +134,7 @@ public abstract class GISModelWriter extends AbstractModelWriter {
* PARAMS[pid].getParams(oid); id++; }
*/
sortPreds[pid] = new ComparablePredicate(PRED_LABELS[pid],
- activeOutcomes, activeParams);
+ predkeys, activeParams);
}
Arrays.sort(sortPreds);
@@ -143,17 +142,17 @@ public abstract class GISModelWriter extends AbstractModelWriter {
}
protected List<List<ComparablePredicate>> compressOutcomes(ComparablePredicate[] sorted) {
- List<List<ComparablePredicate>> outcomePatterns = new ArrayList<List<ComparablePredicate>>();
+ List<List<ComparablePredicate>> outcomePatterns = new ArrayList<>();
if(sorted.length > 0) {
ComparablePredicate cp = sorted[0];
- List<ComparablePredicate> newGroup = new ArrayList<ComparablePredicate>();
+ List<ComparablePredicate> newGroup = new ArrayList<>();
for (int i = 0; i < sorted.length; i++) {
if (cp.compareTo(sorted[i]) == 0) {
newGroup.add(sorted[i]);
} else {
cp = sorted[i];
outcomePatterns.add(newGroup);
- newGroup = new ArrayList<ComparablePredicate>();
+ newGroup = new ArrayList<>();
newGroup.add(sorted[i]);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/OldFormatGISModelReader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/OldFormatGISModelReader.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/OldFormatGISModelReader.java
index 31d6041..8315692 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/OldFormatGISModelReader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/OldFormatGISModelReader.java
@@ -35,7 +35,7 @@ import opennlp.tools.ml.model.Context;
* which stores the parameters.
*/
public class OldFormatGISModelReader extends PlainTextGISModelReader {
- DataInputStream paramsInput;
+ private DataInputStream paramsInput;
/**
* Constructor which takes the name of the model without any suffixes, such as
@@ -68,9 +68,7 @@ public class OldFormatGISModelReader extends PlainTextGISModelReader {
for (int i = 0; i < outcomePatterns.length; i++) {
// construct outcome pattern
int[] outcomePattern = new int[outcomePatterns[i].length - 1];
- for (int k = 1; k < outcomePatterns[i].length; k++) {
- outcomePattern[k - 1] = outcomePatterns[i][k];
- }
+ System.arraycopy(outcomePatterns[i], 1, outcomePattern, 0, outcomePatterns[i].length - 1);
// populate parameters for each context which uses this outcome pattern.
for (int j = 0; j < outcomePatterns[i][0]; j++) {
double[] contextParameters = new double[outcomePatterns[i].length - 1];
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/PlainTextGISModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/PlainTextGISModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/PlainTextGISModelWriter.java
index 3ef5c48..af94871 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/PlainTextGISModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/PlainTextGISModelWriter.java
@@ -21,20 +21,18 @@ package opennlp.tools.ml.maxent.io;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;
-
import opennlp.tools.ml.model.AbstractModel;
/**
* Model writer that saves models in plain text format.
*/
public class PlainTextGISModelWriter extends GISModelWriter {
- BufferedWriter output;
+ private BufferedWriter output;
/**
* Constructor which takes a GISModel and a File and prepares itself to
@@ -45,7 +43,7 @@ public class PlainTextGISModelWriter extends GISModelWriter {
* @param f The File in which the model is to be persisted.
*/
public PlainTextGISModelWriter (AbstractModel model, File f)
- throws IOException, FileNotFoundException {
+ throws IOException {
super(model);
if (f.getName().endsWith(".gz")) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/SuffixSensitiveGISModelReader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/SuffixSensitiveGISModelReader.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/SuffixSensitiveGISModelReader.java
index 4bc6fd6..b6ba705 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/SuffixSensitiveGISModelReader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/io/SuffixSensitiveGISModelReader.java
@@ -76,8 +76,7 @@ public class SuffixSensitiveGISModelReader extends GISModelReader {
* inspect a few of them as plain text files.
*/
public static void main(String[] args) throws IOException {
- AbstractModel m = new SuffixSensitiveGISModelReader(new File(args[0]))
- .getModel();
+ AbstractModel m = new SuffixSensitiveGISModelReader(new File(args[0])).getModel();
new SuffixSensitiveGISModelWriter(m, new File(args[1])).persist();
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/ParallelNegLogLikelihood.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/ParallelNegLogLikelihood.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/ParallelNegLogLikelihood.java
index 00cb55b..bbe8d1b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/ParallelNegLogLikelihood.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/ParallelNegLogLikelihood.java
@@ -35,7 +35,7 @@ import opennlp.tools.ml.model.DataIndexer;
public class ParallelNegLogLikelihood extends NegLogLikelihood {
// Number of threads
- int threads;
+ private int threads;
// Partial value of negative log-likelihood to be computed by each thread
private double[] negLogLikelihoodThread;
@@ -113,7 +113,7 @@ public class ParallelNegLogLikelihood extends NegLogLikelihood {
ParallelNegLogLikelihood.class,
int.class, int.class, int.class, double[].class);
- List<Future<?>> futures = new ArrayList<Future<?>>();
+ List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < threads; i++) {
if (i != threads - 1)
futures.add(executor.submit(
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/model/MaxentModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/MaxentModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/MaxentModel.java
index e8f9e28..a5fd184 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/MaxentModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/MaxentModel.java
@@ -33,7 +33,7 @@ public interface MaxentModel {
* outcomes, all of which sum to 1.
*
**/
- public double[] eval(String[] context);
+ double[] eval(String[] context);
/**
* Evaluates a context.
@@ -44,7 +44,7 @@ public interface MaxentModel {
* outcomes, all of which sum to 1.
* @return an array of the probabilities for each of the different outcomes, all of which sum to 1.
**/
- public double[] eval(String[] context, double probs[]);
+ double[] eval(String[] context, double probs[]);
/**
* Evaluates a contexts with the specified context values.
@@ -53,7 +53,7 @@ public interface MaxentModel {
* @param values The values associated with each context.
* @return an array of the probabilities for each of the different outcomes, all of which sum to 1.
*/
- public double[] eval(String[] context, float[] values);
+ double[] eval(String[] context, float[] values);
/**
* Simple function to return the outcome associated with the index
@@ -64,7 +64,7 @@ public interface MaxentModel {
* method.
* @return the String name of the best outcome
**/
- public String getBestOutcome(double[] outcomes);
+ String getBestOutcome(double[] outcomes);
/**
* Return a string matching all the outcome names with all the
@@ -79,7 +79,7 @@ public interface MaxentModel {
* for each one.
**/
// TODO: This should be removed, can't be used anyway without format spec
- public String getAllOutcomes(double[] outcomes);
+ String getAllOutcomes(double[] outcomes);
/**
* Gets the String name of the outcome associated with the index
@@ -89,7 +89,7 @@ public interface MaxentModel {
* desired.
* @return the String name of the outcome
**/
- public String getOutcome(int i);
+ String getOutcome(int i);
/**
* Gets the index associated with the String name of the given
@@ -100,9 +100,9 @@ public interface MaxentModel {
* @return the index if the given outcome label exists for this
* model, -1 if it does not.
**/
- public int getIndex(String outcome);
+ int getIndex(String outcome);
- /**
+ /*
* Returns the data structures relevant to storing the model.
**/
// public Object[] getDataStructures();
@@ -110,6 +110,6 @@ public interface MaxentModel {
/** Returns the number of outcomes for this model.
* @return The number of outcomes.
**/
- public int getNumOutcomes();
+ int getNumOutcomes();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
index 5b9b1cb..a01b050 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassDataIndexer.java
@@ -68,7 +68,7 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
*/
public OnePassDataIndexer(ObjectStream<Event> eventStream, int cutoff, boolean sort)
throws IOException {
- Map<String, Integer> predicateIndex = new HashMap<String, Integer>();
+ Map<String, Integer> predicateIndex = new HashMap<>();
LinkedList<Event> events;
List<ComparableEvent> eventsToCompare;
@@ -108,9 +108,9 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
*/
private LinkedList<Event> computeEventCounts(ObjectStream<Event> eventStream,
Map<String, Integer> predicatesInOut, int cutoff) throws IOException {
- Set<String> predicateSet = new HashSet<String>();
- Map<String, Integer> counter = new HashMap<String, Integer>();
- LinkedList<Event> events = new LinkedList<Event>();
+ Set<String> predicateSet = new HashSet<>();
+ Map<String, Integer> counter = new HashMap<>();
+ LinkedList<Event> events = new LinkedList<>();
Event ev;
while ((ev = eventStream.read()) != null) {
events.addLast(ev);
@@ -128,12 +128,12 @@ public class OnePassDataIndexer extends AbstractDataIndexer {
protected List<ComparableEvent> index(LinkedList<Event> events,
Map<String, Integer> predicateIndex) {
- Map<String, Integer> omap = new HashMap<String, Integer>();
+ Map<String, Integer> omap = new HashMap<>();
int numEvents = events.size();
int outcomeCount = 0;
- List<ComparableEvent> eventsToCompare = new ArrayList<ComparableEvent>(numEvents);
- List<Integer> indexedContext = new ArrayList<Integer>();
+ List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
+ List<Integer> indexedContext = new ArrayList<>();
for (int eventIndex = 0; eventIndex < numEvents; eventIndex++) {
Event ev = events.removeFirst();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java
index 9a7fb11..ad78172 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/Prior.java
@@ -31,7 +31,7 @@ public interface Prior {
* @param dist An array to be populated with the log of the prior distribution.
* @param context The indices of the contextual predicates for an event.
*/
- public void logPrior(double[] dist, int[] context);
+ void logPrior(double[] dist, int[] context);
/**
* Populates the specified array with the the log of the distribution for the specified context.
@@ -40,7 +40,7 @@ public interface Prior {
* @param context The indices of the contextual predicates for an event.
* @param values The values associated with the context.
*/
- public void logPrior(double[] dist, int[] context, float[] values);
+ void logPrior(double[] dist, int[] context, float[] values);
/**
* Method to specify the label for the outcomes and contexts. This is used to map
@@ -49,5 +49,5 @@ public interface Prior {
* @param outcomeLabels An array of each outcome label.
* @param contextLabels An array of each context label.
*/
- public void setLabels(String[] outcomeLabels, String[] contextLabels);
+ void setLabels(String[] outcomeLabels, String[] contextLabels);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceStream.java
index 7d5dc4d..28bb449 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/SequenceStream.java
@@ -32,6 +32,6 @@ public interface SequenceStream extends ObjectStream<Sequence> {
* @param sequence The sequence to be evaluated.
* @return event array
*/
- public Event[] updateContext(Sequence sequence, AbstractModel model);
+ Event[] updateContext(Sequence sequence, AbstractModel model);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
index 821335b..86fb6a5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/TwoPassDataIndexer.java
@@ -70,7 +70,7 @@ public class TwoPassDataIndexer extends AbstractDataIndexer{
* observed in order to be included in the model.
*/
public TwoPassDataIndexer(ObjectStream<Event> eventStream, int cutoff, boolean sort) throws IOException {
- Map<String,Integer> predicateIndex = new HashMap<String,Integer>();
+ Map<String,Integer> predicateIndex = new HashMap<>();
List<ComparableEvent> eventsToCompare;
System.out.println("Indexing events using cutoff of " + cutoff + "\n");
@@ -118,10 +118,11 @@ public class TwoPassDataIndexer extends AbstractDataIndexer{
* @param predicatesInOut a <code>TObjectIntHashMap</code> value
* @param cutoff an <code>int</code> value
*/
- private int computeEventCounts(ObjectStream<Event> eventStream, Writer eventStore, Map<String,Integer> predicatesInOut, int cutoff) throws IOException {
- Map<String,Integer> counter = new HashMap<String,Integer>();
+ private int computeEventCounts(ObjectStream<Event> eventStream, Writer eventStore,
+ Map<String,Integer> predicatesInOut, int cutoff) throws IOException {
+ Map<String,Integer> counter = new HashMap<>();
int eventCount = 0;
- Set<String> predicateSet = new HashSet<String>();
+ Set<String> predicateSet = new HashSet<>();
Event ev;
while ((ev = eventStream.read()) != null) {
@@ -142,10 +143,10 @@ public class TwoPassDataIndexer extends AbstractDataIndexer{
}
private List<ComparableEvent> index(int numEvents, ObjectStream<Event> es, Map<String,Integer> predicateIndex) throws IOException {
- Map<String,Integer> omap = new HashMap<String,Integer>();
+ Map<String,Integer> omap = new HashMap<>();
int outcomeCount = 0;
- List<ComparableEvent> eventsToCompare = new ArrayList<ComparableEvent>(numEvents);
- List<Integer> indexedContext = new ArrayList<Integer>();
+ List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
+ List<Integer> indexedContext = new ArrayList<>();
Event ev;
while ((ev = es.read()) != null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/LogProbabilities.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/LogProbabilities.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/LogProbabilities.java
index dd91ff9..0814cf3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/LogProbabilities.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/LogProbabilities.java
@@ -159,7 +159,7 @@ public class LogProbabilities<T> extends Probabilities<T> {
public void discardCountsBelow(double i) {
i = Math.log(i);
- ArrayList<T> labelsToRemove = new ArrayList<T>();
+ ArrayList<T> labelsToRemove = new ArrayList<>();
for (T label : map.keySet()) {
Double sum = map.get(label);
if (sum == null) sum = Double.NEGATIVE_INFINITY;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
index 05503fa..35eaa26 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
@@ -91,7 +91,7 @@ public class NaiveBayesModel extends AbstractModel {
}
public static double[] eval(int[] context, float[] values, double[] prior, EvalParameters model, boolean normalize) {
- Probabilities<Integer> probabilities = new LogProbabilities<Integer>();
+ Probabilities<Integer> probabilities = new LogProbabilities<>();
Context[] params = model.getParams();
double[] outcomeTotals = model instanceof NaiveBayesEvalParameters ? ((NaiveBayesEvalParameters) model).getOutcomeTotals() : new double[prior.length];
long vocabulary = model instanceof NaiveBayesEvalParameters ? ((NaiveBayesEvalParameters) model).getVocabulary() : 0;
@@ -121,8 +121,7 @@ public class NaiveBayesModel extends AbstractModel {
}
for (int i = 0; i < outcomeTotals.length; ++i) {
double numerator = outcomeTotals[i];
- double denominator = total;
- probabilities.addIn(i, numerator / denominator, 1);
+ probabilities.addIn(i, numerator / total, 1);
}
for (int i = 0; i < outcomeTotals.length; ++i) {
prior[i] = probabilities.get(i);
@@ -141,9 +140,8 @@ public class NaiveBayesModel extends AbstractModel {
private static double getSmoothedProbability(double numerator, double denominator, double vocabulary) {
final double delta = 0.05; // Lidstone smoothing
- final double featureVocabularySize = vocabulary;
- return 1.0 * (numerator + delta) / (denominator + delta * featureVocabularySize);
+ return 1.0 * (numerator + delta) / (denominator + delta * vocabulary);
}
public static void main(String[] args) throws java.io.IOException {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java
index eb17aef..d1a9263 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModelWriter.java
@@ -98,15 +98,15 @@ public abstract class NaiveBayesModelWriter extends AbstractModelWriter {
protected List<List<ComparablePredicate>> computeOutcomePatterns(ComparablePredicate[] sorted) {
ComparablePredicate cp = sorted[0];
- List<List<ComparablePredicate>> outcomePatterns = new ArrayList<List<ComparablePredicate>>();
- List<ComparablePredicate> newGroup = new ArrayList<ComparablePredicate>();
+ List<List<ComparablePredicate>> outcomePatterns = new ArrayList<>();
+ List<ComparablePredicate> newGroup = new ArrayList<>();
for (ComparablePredicate predicate : sorted) {
if (cp.compareTo(predicate) == 0) {
newGroup.add(predicate);
} else {
cp = predicate;
outcomePatterns.add(newGroup);
- newGroup = new ArrayList<ComparablePredicate>();
+ newGroup = new ArrayList<>();
newGroup.add(predicate);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
index c3870f9..b39b659 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
@@ -147,7 +147,7 @@ public class NaiveBayesTrainer extends AbstractEventTrainer {
for (int oi = 0; oi < numOutcomes; oi++)
allOutcomesPattern[oi] = oi;
- /** Stores the estimated parameter value of each predicate during iteration. */
+ /* Stores the estimated parameter value of each predicate during iteration. */
MutableContext[] params = new MutableContext[numPreds];
for (int pi = 0; pi < numPreds; pi++) {
params[pi] = new MutableContext(allOutcomesPattern, new double[numOutcomes]);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/PlainTextNaiveBayesModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/PlainTextNaiveBayesModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/PlainTextNaiveBayesModelWriter.java
index 7047419..5aa0682 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/PlainTextNaiveBayesModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/PlainTextNaiveBayesModelWriter.java
@@ -21,20 +21,18 @@ package opennlp.tools.ml.naivebayes;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;
-
import opennlp.tools.ml.model.AbstractModel;
/**
* Model writer that saves models in plain text format.
*/
public class PlainTextNaiveBayesModelWriter extends NaiveBayesModelWriter {
- BufferedWriter output;
+ private BufferedWriter output;
/**
* Constructor which takes a NaiveBayesModel and a File and prepares itself to
@@ -45,7 +43,7 @@ public class PlainTextNaiveBayesModelWriter extends NaiveBayesModelWriter {
* @param f The File in which the model is to be persisted.
*/
public PlainTextNaiveBayesModelWriter(AbstractModel model, File f)
- throws IOException, FileNotFoundException {
+ throws IOException {
super(model);
if (f.getName().endsWith(".gz")) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/Probabilities.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/Probabilities.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/Probabilities.java
index 4c53737..394a4c6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/Probabilities.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/Probabilities.java
@@ -26,7 +26,7 @@ import java.util.*;
*
*/
public abstract class Probabilities<T> {
- protected HashMap<T, Double> map = new HashMap<T, Double>();
+ protected HashMap<T, Double> map = new HashMap<>();
protected transient boolean isNormalised = false;
protected Map<T, Double> normalised;
@@ -159,7 +159,7 @@ public abstract class Probabilities<T> {
}
protected Map<T, Double> createMapDataStructure() {
- return new HashMap<T, Double>();
+ return new HashMap<>();
}
/**
@@ -190,7 +190,7 @@ public abstract class Probabilities<T> {
}
public void discardCountsBelow(double i) {
- ArrayList<T> labelsToRemove = new ArrayList<T>();
+ ArrayList<T> labelsToRemove = new ArrayList<>();
for (T label : map.keySet()) {
Double sum = map.get(label);
if (sum == null) sum = 0.0;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/BilouCodec.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/BilouCodec.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/BilouCodec.java
index 633ba35..7e8508a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/BilouCodec.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/BilouCodec.java
@@ -37,7 +37,7 @@ public class BilouCodec implements SequenceCodec<String> {
public Span[] decode(List<String> c) {
int start = -1;
int end = -1;
- List<Span> spans = new ArrayList<Span>(c.size());
+ List<Span> spans = new ArrayList<>(c.size());
for (int li = 0; li < c.size(); li++) {
String chunkTag = c.get(li);
if (chunkTag.endsWith(BioCodec.START)) {
@@ -57,9 +57,6 @@ public class BilouCodec implements SequenceCodec<String> {
else if (chunkTag.endsWith(UNIT)) {
spans.add(new Span(li, li + 1, BioCodec.extractNameType(c.get(li))));
}
- else if (chunkTag.endsWith(BioCodec.OTHER)) {
- // in this case do nothing
- }
}
return spans.toArray(new Span[spans.size()]);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/BioCodec.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/BioCodec.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/BioCodec.java
index 1367f28..1502081 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/BioCodec.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/BioCodec.java
@@ -33,11 +33,10 @@ public class BioCodec implements SequenceCodec<String> {
private static final Pattern typedOutcomePattern = Pattern.compile("(.+)-\\w+");
- static final String extractNameType(String outcome) {
+ static String extractNameType(String outcome) {
Matcher matcher = typedOutcomePattern.matcher(outcome);
if(matcher.matches()) {
- String nameType = matcher.group(1);
- return nameType;
+ return matcher.group(1);
}
return null;
@@ -46,7 +45,7 @@ public class BioCodec implements SequenceCodec<String> {
public Span[] decode(List<String> c) {
int start = -1;
int end = -1;
- List<Span> spans = new ArrayList<Span>(c.size());
+ List<Span> spans = new ArrayList<>(c.size());
for (int li = 0; li < c.size(); li++) {
String chunkTag = c.get(li);
if (chunkTag.endsWith(BioCodec.START)) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
index a8d249f..108efa6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
@@ -63,7 +63,7 @@ public class DictionaryNameFinder implements TokenNameFinder {
}
public Span[] find(String[] textTokenized) {
- List<Span> namesFound = new LinkedList<Span>();
+ List<Span> namesFound = new LinkedList<>();
for (int offsetFrom = 0; offsetFrom < textTokenized.length; offsetFrom++) {
Span nameFound = null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/DocumentNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/DocumentNameFinder.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/DocumentNameFinder.java
index 130699a..c1c1b2d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/DocumentNameFinder.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/DocumentNameFinder.java
@@ -36,6 +36,6 @@ public interface DocumentNameFinder {
* @param document An array of tokens for each sentence of a document.
* @return The token spans for each sentence of the specified document.
*/
- public abstract Span[][] find(String[][] document);
+ Span[][] find(String[][] document);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameContextGenerator.java
index b10db43..be3ee2e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameContextGenerator.java
@@ -30,18 +30,18 @@ public interface NameContextGenerator extends BeamSearchContextGenerator<String>
* Adds a feature generator to this set of feature generators.
* @param generator The feature generator to add.
*/
- public void addFeatureGenerator(AdaptiveFeatureGenerator generator);
+ void addFeatureGenerator(AdaptiveFeatureGenerator generator);
/**
* Informs all the feature generators for a name finder that the specified tokens have been classified with the coorisponds set of specified outcomes.
* @param tokens The tokens of the sentence or other text unit which has been processed.
* @param outcomes The outcomes associated with the specified tokens.
*/
- public void updateAdaptiveData(String[] tokens, String[] outcomes);
+ void updateAdaptiveData(String[] tokens, String[] outcomes);
/**
* Informs all the feature generators for a name finder that the context of the adaptive data (typically a document) is no longer valid.
*/
- public void clearAdaptiveData();
+ void clearAdaptiveData();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
index b20971f..65b01c5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderEventStream.java
@@ -39,8 +39,6 @@ public class NameFinderEventStream extends opennlp.tools.util.AbstractEventStrea
private AdditionalContextFeatureGenerator additionalContextFeatureGenerator = new AdditionalContextFeatureGenerator();
- private String type;
-
private SequenceCodec<String> codec;
/**
@@ -61,10 +59,11 @@ public class NameFinderEventStream extends opennlp.tools.util.AbstractEventStrea
this.contextGenerator = contextGenerator;
this.contextGenerator.addFeatureGenerator(new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));
+ String type1;
if (type != null)
- this.type = type;
+ type1 = type;
else
- this.type = "default";
+ type1 = "default";
}
public NameFinderEventStream(ObjectStream<NameSample> dataStream) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
index 1824909..00e001c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-
import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.EventModelSequenceTrainer;
import opennlp.tools.ml.EventTrainer;
@@ -54,7 +53,6 @@ import opennlp.tools.util.featuregen.SentenceFeatureGenerator;
import opennlp.tools.util.featuregen.TokenClassFeatureGenerator;
import opennlp.tools.util.featuregen.TokenFeatureGenerator;
import opennlp.tools.util.featuregen.WindowFeatureGenerator;
-import opennlp.tools.util.model.ModelUtil;
/**
* Class for creating a maximum-entropy-based name finder.
@@ -95,7 +93,7 @@ public class NameFinderME implements TokenNameFinder {
}
@Deprecated
- /**
+ /*
* @deprecated the default feature generation is now always included in the models and loaded
* if not by the factory. Subclasses using this methods should do the same.
*/
@@ -253,7 +251,7 @@ public class NameFinderME implements TokenNameFinder {
beamSize = Integer.parseInt(beamSizeString);
}
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
MaxentModel nameFinderModel = null;
@@ -327,7 +325,7 @@ public class NameFinderME implements TokenNameFinder {
beamSize = Integer.parseInt(beamSizeString);
}
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
AdaptiveFeatureGenerator featureGenerator;
@@ -418,11 +416,10 @@ public class NameFinderME implements TokenNameFinder {
* @param outcome the outcome
* @return the name type, or null if not set
*/
- static final String extractNameType(String outcome) {
+ static String extractNameType(String outcome) {
Matcher matcher = typedOutcomePattern.matcher(outcome);
if (matcher.matches()) {
- String nameType = matcher.group(1);
- return nameType;
+ return matcher.group(1);
}
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
index 1fba758..e1dc801 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
@@ -55,8 +55,8 @@ public class NameSample {
names = new Span[0];
}
- this.sentence = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(sentence)));
- this.names = Collections.unmodifiableList(new ArrayList<Span>(Arrays.asList(names)));
+ this.sentence = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(sentence)));
+ this.names = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(names)));
if (additionalContext != null) {
this.additionalContext = new String[additionalContext.length][];
@@ -215,8 +215,8 @@ public class NameSample {
throws IOException {
String[] parts = WhitespaceTokenizer.INSTANCE.tokenize(taggedTokens);
- List<String> tokenList = new ArrayList<String>(parts.length);
- List<Span> nameList = new ArrayList<Span>();
+ List<String> tokenList = new ArrayList<>(parts.length);
+ List<Span> nameList = new ArrayList<>();
String nameType = defaultType;
int startIndex = -1;
[15/50] [abbrv] opennlp git commit: Remove unnecessary static
modifier from enum
Posted by jo...@apache.org.
Remove unnecessary static modifier from enum
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/cd6b11dc
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/cd6b11dc
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/cd6b11dc
Branch: refs/heads/889
Commit: cd6b11dc616de890f3072876aeff656ed66eefd1
Parents: 9e1e77d
Author: J�rn Kottmann <jo...@apache.org>
Authored: Fri Oct 28 21:46:15 2016 +0200
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:32 2016 +0100
----------------------------------------------------------------------
.../main/java/opennlp/tools/namefind/RegexNameFinderFactory.java | 2 +-
.../main/java/opennlp/tools/tokenize/DetokenizationDictionary.java | 2 +-
.../src/main/java/opennlp/tools/tokenize/Detokenizer.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/cd6b11dc/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
index 0c13577..b72d3a9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
@@ -107,7 +107,7 @@ public class RegexNameFinderFactory {
public String getType();
}
- public static enum DEFAULT_REGEX_NAME_FINDER implements RegexAble {
+ public enum DEFAULT_REGEX_NAME_FINDER implements RegexAble {
USA_PHONE_NUM {
@Override
http://git-wip-us.apache.org/repos/asf/opennlp/blob/cd6b11dc/opennlp-tools/src/main/java/opennlp/tools/tokenize/DetokenizationDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/DetokenizationDictionary.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/DetokenizationDictionary.java
index 44f38c5..2fbb4a3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/DetokenizationDictionary.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/DetokenizationDictionary.java
@@ -33,7 +33,7 @@ import opennlp.tools.util.StringList;
public class DetokenizationDictionary {
- public static enum Operation {
+ public enum Operation {
/**
* Attaches the token to the token on the right side.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/cd6b11dc/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
index ee90720..c962aa4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/Detokenizer.java
@@ -27,7 +27,7 @@ public interface Detokenizer {
* This enum contains an operation for every token to merge the
* tokens together to their detokenized form.
*/
- public static enum DetokenizationOperation {
+ public enum DetokenizationOperation {
/**
* The current token should be attached to the begin token on the right side.
*/
[31/50] [abbrv] opennlp git commit: OPENNLP-894: Switch from Java 7
to Java 8 and enforce Maven 3.3.9
Posted by jo...@apache.org.
OPENNLP-894: Switch from Java 7 to Java 8 and enforce Maven 3.3.9
This closes #10
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/4f279085
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/4f279085
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/4f279085
Branch: refs/heads/889
Commit: 4f2790853b3bf1c39cd98683fe84859fb35e36b5
Parents: afc6b65
Author: smarthi <sm...@apache.org>
Authored: Mon Dec 19 23:25:10 2016 -0500
Committer: Joern Kottmann <jo...@apache.org>
Committed: Tue Dec 20 14:45:15 2016 +0100
----------------------------------------------------------------------
opennlp-brat-annotator/pom.xml | 3 -
opennlp-distr/README | 6 +-
opennlp-distr/pom.xml | 4 +-
opennlp-distr/src/main/assembly/bin.xml | 3 -
opennlp-distr/src/main/assembly/src.xml | 1 -
opennlp-morfologik-addon/pom.xml | 19 ++---
opennlp-tools/pom.xml | 11 +--
opennlp-uima/pom.xml | 17 +----
.../uima/AnnotatorsInitializationTest.java | 10 +--
.../uima/dictionary/DictionaryResourceTest.java | 2 +-
.../test/resources/test-descriptors/Chunker.xml | 1 -
.../test-descriptors/DateNameFinder.xml | 1 -
.../test-descriptors/DictionaryNameFinder.xml | 3 +-
.../test-descriptors/LocationNameFinder.xml | 1 -
.../test-descriptors/MoneyNameFinder.xml | 1 -
.../test-descriptors/OrganizationNameFinder.xml | 1 -
.../test-descriptors/PercentageNameFinder.xml | 1 -
.../test-descriptors/PersonNameFinder.xml | 1 -
.../resources/test-descriptors/PosTagger.xml | 1 -
.../test-descriptors/SentenceDetector.xml | 1 -
.../test-descriptors/TimeNameFinder.xml | 1 -
.../resources/test-descriptors/Tokenizer.xml | 1 -
pom.xml | 73 +++++++++++++++++---
23 files changed, 83 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-brat-annotator/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-brat-annotator/pom.xml b/opennlp-brat-annotator/pom.xml
index b7e7301..5f77bb7 100644
--- a/opennlp-brat-annotator/pom.xml
+++ b/opennlp-brat-annotator/pom.xml
@@ -23,7 +23,6 @@
</parent>
<artifactId>opennlp-brat-annotator</artifactId>
- <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache OpenNLP Brat Annotator</name>
@@ -60,13 +59,11 @@
<dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-tools</artifactId>
- <version>1.6.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-distr/README
----------------------------------------------------------------------
diff --git a/opennlp-distr/README b/opennlp-distr/README
index a0ad5e4..0ecfaec 100644
--- a/opennlp-distr/README
+++ b/opennlp-distr/README
@@ -18,7 +18,7 @@ What is new in Apache OpenNLP ${pom.version}
This release introduces many new features, improvements and bug fixes. The API
has been improved for a better consistency and 1.4 deprecated methods were
-removed. Now Java 1.7 is required.
+removed. Now Java 1.8 is required.
Additionally the release contains the following noteworthy changes:
@@ -39,6 +39,6 @@ notes.
Requirements
------------
-Java 1.7 is required to run OpenNLP
-Maven 3.0.0 is required for building it
+Java 1.8 is required to run OpenNLP
+Maven 3.3.9 is required for building it
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-distr/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-distr/pom.xml b/opennlp-distr/pom.xml
index 01f2006..e5243cf 100644
--- a/opennlp-distr/pom.xml
+++ b/opennlp-distr/pom.xml
@@ -37,12 +37,10 @@
<dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-tools</artifactId>
- <version>1.6.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.opennlp</groupId>
<artifactId>opennlp-uima</artifactId>
- <version>1.6.1-SNAPSHOT</version>
</dependency>
</dependencies>
@@ -114,7 +112,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-changes-plugin</artifactId>
- <version>2.9</version>
+ <version>2.12.1</version>
<executions>
<execution>
<id>default-cli</id>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-distr/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml
index 2fd5d7f..926a9eb 100644
--- a/opennlp-distr/src/main/assembly/bin.xml
+++ b/opennlp-distr/src/main/assembly/bin.xml
@@ -46,14 +46,12 @@
<fileSets>
<fileSet>
<directory>src/main/readme</directory>
- <outputDirectory></outputDirectory>
<fileMode>644</fileMode>
<directoryMode>755</directoryMode>
</fileSet>
<fileSet>
<directory>.</directory>
- <outputDirectory></outputDirectory>
<filtered>true</filtered>
<fileMode>644</fileMode>
<directoryMode>755</directoryMode>
@@ -65,7 +63,6 @@
<fileSet>
<directory>target</directory>
- <outputDirectory></outputDirectory>
<fileMode>644</fileMode>
<directoryMode>755</directoryMode>
<includes>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-distr/src/main/assembly/src.xml
----------------------------------------------------------------------
diff --git a/opennlp-distr/src/main/assembly/src.xml b/opennlp-distr/src/main/assembly/src.xml
index cdcc9d3..1dea96b 100644
--- a/opennlp-distr/src/main/assembly/src.xml
+++ b/opennlp-distr/src/main/assembly/src.xml
@@ -27,7 +27,6 @@
<fileSets>
<fileSet>
<directory>../</directory>
- <outputDirectory></outputDirectory>
<excludes>
<exclude>**/target/**</exclude>
<exclude>**/.*/**</exclude>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-morfologik-addon/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/pom.xml b/opennlp-morfologik-addon/pom.xml
index 71d2c53..da9b728 100644
--- a/opennlp-morfologik-addon/pom.xml
+++ b/opennlp-morfologik-addon/pom.xml
@@ -2,9 +2,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.opennlp</groupId>
+ <parent>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp</artifactId>
+ <version>1.6.1-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
<artifactId>opennlp-morfologik-addon</artifactId>
- <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache OpenNLP Morfologik Addon</name>
@@ -12,15 +17,6 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
- <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
@@ -101,7 +97,6 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.8.1</version>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-tools/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-tools/pom.xml b/opennlp-tools/pom.xml
index db0d151..9afaff4 100644
--- a/opennlp-tools/pom.xml
+++ b/opennlp-tools/pom.xml
@@ -55,6 +55,7 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
@@ -116,7 +117,7 @@
<configuration>
<instructions>
<Bundle-Activator>opennlp.tools.util.ext.OSGiExtensionLoader</Bundle-Activator>
- <Bundle-RequiredExecutionEnvironment>JavaSE-1.7</Bundle-RequiredExecutionEnvironment>
+ <Bundle-RequiredExecutionEnvironment>JavaSE-1.8</Bundle-RequiredExecutionEnvironment>
<Export-Package>
!opennlp.tools.cmdline.*,
opennlp.tools.*
@@ -177,14 +178,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/pom.xml b/opennlp-uima/pom.xml
index 2347053..11bd95f 100644
--- a/opennlp-uima/pom.xml
+++ b/opennlp-uima/pom.xml
@@ -83,22 +83,7 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>true</skipTests>
- <argLine>-Xmx512m</argLine>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.7</source>
- <target>1.7</target>
- </configuration>
- </plugin>
+
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/java/opennlp/uima/AnnotatorsInitializationTest.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/java/opennlp/uima/AnnotatorsInitializationTest.java b/opennlp-uima/src/test/java/opennlp/uima/AnnotatorsInitializationTest.java
index ced03c8..7e77a6d 100644
--- a/opennlp-uima/src/test/java/opennlp/uima/AnnotatorsInitializationTest.java
+++ b/opennlp-uima/src/test/java/opennlp/uima/AnnotatorsInitializationTest.java
@@ -16,6 +16,8 @@
*/
package opennlp.uima;
+import java.io.File;
+import java.io.IOException;
import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.cas.CAS;
@@ -24,10 +26,6 @@ import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.ResourceSpecifier;
import org.apache.uima.util.InvalidXMLException;
import org.apache.uima.util.XMLInputSource;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
import static org.junit.Assert.fail;
@@ -38,7 +36,9 @@ public class AnnotatorsInitializationTest {
private static final String PATHNAME = "src/test/resources/test-descriptors/";
- @Test
+ // TODO: This test requires the SourceForge models, or other models to run,
+ // but they are missing due to license issues since the project was migrated to Apache
+ //@Test
public void testInitializationExecutionAndReconfigure() {
File f = new File(PATHNAME);
for (String descName : f.list(new FileUtil.ExtFilenameFilter("xml"))) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/java/opennlp/uima/dictionary/DictionaryResourceTest.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/java/opennlp/uima/dictionary/DictionaryResourceTest.java b/opennlp-uima/src/test/java/opennlp/uima/dictionary/DictionaryResourceTest.java
index 40d24c7..2afb97a 100644
--- a/opennlp-uima/src/test/java/opennlp/uima/dictionary/DictionaryResourceTest.java
+++ b/opennlp-uima/src/test/java/opennlp/uima/dictionary/DictionaryResourceTest.java
@@ -43,7 +43,7 @@ import static org.junit.Assert.*;
public class DictionaryResourceTest {
- private static final String PATHNAME = "opennlp-uima/src/test/resources/test-descriptors/";
+ private static final String PATHNAME = "src/test/resources/test-descriptors/";
private static AnalysisEngine AE;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/Chunker.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/Chunker.xml b/opennlp-uima/src/test/resources/test-descriptors/Chunker.xml
index dd48498..ecca12e 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/Chunker.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/Chunker.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.chunker.Chunker</annotatorImplementationName>
<analysisEngineMetaData>
<name>Chunker</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/DateNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/DateNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/DateNameFinder.xml
index 8f58e3b..d619160 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/DateNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/DateNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Date Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/DictionaryNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/DictionaryNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/DictionaryNameFinder.xml
index e063754..97e19b2 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/DictionaryNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/DictionaryNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.DictionaryNameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Dictionary Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
@@ -109,7 +108,7 @@
<name>NameFinderDictionary</name>
<description/>
<fileResourceSpecifier>
- <fileUrl>file:opennlp-uima/src/test/resources/dictionary.dic</fileUrl>
+ <fileUrl>file:src/test/resources/dictionary.dic</fileUrl>
</fileResourceSpecifier>
<implementationName>opennlp.uima.dictionary.DictionaryResourceImpl</implementationName>
</externalResource>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/LocationNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/LocationNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/LocationNameFinder.xml
index f6fdaeb..51f5079 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/LocationNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/LocationNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Location Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/MoneyNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/MoneyNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/MoneyNameFinder.xml
index c5b9207..8df918d 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/MoneyNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/MoneyNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Money Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/OrganizationNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/OrganizationNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/OrganizationNameFinder.xml
index c72ff0c..0afe1fc 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/OrganizationNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/OrganizationNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Organization Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/PercentageNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/PercentageNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/PercentageNameFinder.xml
index 8235d64..695d58e 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/PercentageNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/PercentageNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Percentage Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/PersonNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/PersonNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/PersonNameFinder.xml
index 19e916d..659a3f4 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/PersonNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/PersonNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Person Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/PosTagger.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/PosTagger.xml b/opennlp-uima/src/test/resources/test-descriptors/PosTagger.xml
index 5fe00b6..e3eba91 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/PosTagger.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/PosTagger.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.postag.POSTagger</annotatorImplementationName>
<analysisEngineMetaData>
<name>POS Tagger</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/SentenceDetector.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/SentenceDetector.xml b/opennlp-uima/src/test/resources/test-descriptors/SentenceDetector.xml
index 2025fa4..2a02034 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/SentenceDetector.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/SentenceDetector.xml
@@ -26,7 +26,6 @@
<annotatorImplementationName>opennlp.uima.sentdetect.SentenceDetector</annotatorImplementationName>
<analysisEngineMetaData>
<name>Sentence Detector</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/TimeNameFinder.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/TimeNameFinder.xml b/opennlp-uima/src/test/resources/test-descriptors/TimeNameFinder.xml
index 3f7ed0b..846b5a8 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/TimeNameFinder.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/TimeNameFinder.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.namefind.NameFinder</annotatorImplementationName>
<analysisEngineMetaData>
<name>Time Name Finder</name>
- <description></description>
<version>1.5.2-incubating</version>
<vendor>Apache Software Foundation</vendor>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/opennlp-uima/src/test/resources/test-descriptors/Tokenizer.xml
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/test/resources/test-descriptors/Tokenizer.xml b/opennlp-uima/src/test/resources/test-descriptors/Tokenizer.xml
index cee2a94..27f8480 100644
--- a/opennlp-uima/src/test/resources/test-descriptors/Tokenizer.xml
+++ b/opennlp-uima/src/test/resources/test-descriptors/Tokenizer.xml
@@ -25,7 +25,6 @@
<annotatorImplementationName>opennlp.uima.tokenize.Tokenizer</annotatorImplementationName>
<analysisEngineMetaData>
<name>Tokenizer</name>
- <description></description>
<version>${pom.version}</version>
<vendor>Apache Software Foundation</vendor>
<configurationParameters>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/4f279085/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 91e5043..89d0206 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
- <version>16</version>
+ <version>18</version>
<relativePath />
</parent>
@@ -36,10 +36,6 @@
<name>Apache OpenNLP Reactor</name>
- <prerequisites>
- <maven>3.0</maven>
- </prerequisites>
-
<scm>
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/opennlp.git</connection>
<url>https://git-wip-us.apache.org/repos/asf?p=opennlp.git</url>
@@ -87,12 +83,33 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.8.1</version>
+ <version>${junit.version}</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <artifactId>opennlp-tools</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <artifactId>opennlp-uima</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <version>${project.version}</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
+ <properties>
+ <!-- Build Properties -->
+ <java.version>1.8</java.version>
+ <maven.version>3.3.9</maven.version>
+ <enforcer.plugin.version>1.4.1</enforcer.plugin.version>
+ <junit.version>4.12</junit.version>
+ </properties>
+
<build>
<pluginManagement>
<plugins>
@@ -110,9 +127,7 @@
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
- <!-- Version 2.3.5 breaks backward compatibility with java 5,
- but the build must run on java 5! -->
- <version>2.3.4</version>
+ <version>3.2.0</version>
</plugin>
</plugins>
</pluginManagement>
@@ -120,9 +135,10 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
+ <version>3.3</version>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>${java.version}</source>
+ <target>${java.version}</target>
<compilerArgument>-Xlint</compilerArgument>
</configuration>
</plugin>
@@ -189,6 +205,41 @@
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>${enforcer.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>enforce-java</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <requireJavaVersion>
+ <message>Java 8 or higher is required to compile this module</message>
+ <version>[${java.version},)</version>
+ </requireJavaVersion>
+ <requireMavenVersion>
+ <message>Maven 3.3.9 or higher is required to compile this module</message>
+ <version>[${maven.version},)</version>
+ </requireMavenVersion>
+ </rules>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>false</skipTests>
+ <argLine>-Xmx512m</argLine>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
[22/50] [abbrv] opennlp git commit: Replace == with equals for string
compare
Posted by jo...@apache.org.
Replace == with equals for string compare
Comparing string referencs to check strings are equal
should not be done, in case the strigns somehow change
all those comparisons break. Using equals will result in the
same and only if references are not equal the string will
be compared.
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/6871d116
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/6871d116
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/6871d116
Branch: refs/heads/889
Commit: 6871d1164ef22692141f59bb3176bef4c7777771
Parents: e20dc21
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 20:12:55 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../tools/formats/frenchtreebank/ConstitDocumentHandler.java | 2 +-
opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java | 2 +-
.../src/main/java/opennlp/tools/parser/lang/en/HeadRules.java | 2 +-
.../opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java | 2 +-
.../src/main/java/opennlp/tools/parser/treeinsert/Parser.java | 4 ++--
.../java/opennlp/tools/parser/treeinsert/ParserEventStream.java | 4 ++--
6 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/6871d116/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
index a58b448..2f3cf6a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
@@ -171,7 +171,7 @@ class ConstitDocumentHandler extends DefaultHandler {
Constituent con = cons.get(ci);
String type = con.getLabel();
if (!type.equals(AbstractBottomUpParser.TOP_NODE)) {
- if (type == AbstractBottomUpParser.TOK_NODE) {
+ if (AbstractBottomUpParser.TOK_NODE.equals(type)) {
tokenIndex++;
}
Parse c = new Parse(txt, con.getSpan(), type, 1,tokenIndex);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/6871d116/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java b/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
index 4a57ed1..9bf12f3 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
@@ -883,7 +883,7 @@ public class Parse implements Cloneable, Comparable<Parse> {
Constituent con = cons.get(ci);
String type = con.getLabel();
if (!type.equals(AbstractBottomUpParser.TOP_NODE)) {
- if (type == AbstractBottomUpParser.TOK_NODE) {
+ if (AbstractBottomUpParser.TOK_NODE.equals(type)) {
tokenIndex++;
}
Parse c = new Parse(txt, con.getSpan(), type, 1,tokenIndex);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/6871d116/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
index a8059db..8a5a947 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
@@ -131,7 +131,7 @@ public class HeadRules implements opennlp.tools.parser.HeadRules, GapLabeler, Se
}
public Parse getHead(Parse[] constituents, String type) {
- if (constituents[0].getType() == Parser.TOK_NODE) {
+ if (Parser.TOK_NODE.equals(constituents[0].getType())) {
return null;
}
HeadRule hr;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/6871d116/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
index 5118a08..946fa5c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
@@ -132,7 +132,7 @@ public class AncoraSpanishHeadRules implements opennlp.tools.parser.HeadRules, G
}
public Parse getHead(Parse[] constituents, String type) {
- if (constituents[0].getType() == Parser.TOK_NODE) {
+ if (Parser.TOK_NODE.equals(constituents[0].getType())) {
return null;
}
HeadRule hr;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/6871d116/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
index a06576e..e2c62d7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
@@ -142,8 +142,8 @@ public class Parser extends AbstractBottomUpParser {
public static List<Parse> getRightFrontier(Parse root,Set<String> punctSet) {
List<Parse> rf = new LinkedList<Parse>();
Parse top;
- if (root.getType() == AbstractBottomUpParser.TOP_NODE ||
- root.getType() == AbstractBottomUpParser.INC_NODE) {
+ if (AbstractBottomUpParser.TOP_NODE.equals(root.getType()) ||
+ AbstractBottomUpParser.INC_NODE.equals(root.getType())) {
top = collapsePunctuation(root.getChildren(),punctSet)[0];
}
else {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/6871d116/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
index 6f6c85d..c693ef6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
@@ -273,7 +273,7 @@ public class ParserEventStream extends AbstractParserEventStream {
}
//attach Node
if (attachNode != null) {
- if (attachType == Parser.ATTACH_DAUGHTER) {
+ if (Parser.ATTACH_DAUGHTER.equals(attachType)) {
Parse daughter = currentChunks[ci];
if (debug) System.err.println("daughter attach a="+attachNode.getType()+":"+attachNode+" d="+daughter+" com="+lastChild(chunks[ci], rightFrontier.get(attachNodeIndex)));
attachNode.add(daughter,rules);
@@ -290,7 +290,7 @@ public class ParserEventStream extends AbstractParserEventStream {
}
}
}
- else if (attachType == Parser.ATTACH_SISTER) {
+ else if (Parser.ATTACH_SISTER.equals(attachType)) {
Parse frontierNode = rightFrontier.get(attachNodeIndex);
rightFrontier.set(attachNodeIndex,frontierNode.getParent());
Parse sister = currentChunks[ci];
[23/50] [abbrv] opennlp git commit: Replace while loop with foreach
Posted by jo...@apache.org.
Replace while loop with foreach
The foreach makes the code easier to read
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/e20dc210
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/e20dc210
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/e20dc210
Branch: refs/heads/889
Commit: e20dc210be4bf6d9438e009daf98ac2442a6f722
Parents: c3eeb5c
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 20:02:10 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../cmdline/doccat/DoccatFineGrainedReportListener.java | 12 +++++-------
.../lemmatizer/LemmatizerFineGrainedReportListener.java | 12 +++++-------
.../postag/POSTaggerFineGrainedReportListener.java | 4 +---
3 files changed, 11 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/e20dc210/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
index 932bd6f..e07dc4f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
@@ -275,22 +275,20 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
"% Err", "Precision", "Recall", "F-Measure"));
printLine(tableSize);
- Iterator<String> tagIterator = tags.iterator();
- while (tagIterator.hasNext()) {
- String tag = tagIterator.next();
+ for (String tag : tags) {
int ocurrencies = getTagFrequency(tag);
int errors = getTagErrors(tag);
String rate = MessageFormat.format("{0,number,#.###}", (double) errors
- / ocurrencies);
+ / ocurrencies);
double p = getTagPrecision(tag);
double r = getTagRecall(tag);
double f = getTagFMeasure(tag);
printStream.append(String.format(format, tag, errors, ocurrencies, rate,
- MessageFormat.format("{0,number,#.###}", p > 0 ? p : 0),
- MessageFormat.format("{0,number,#.###}", r > 0 ? r : 0),
- MessageFormat.format("{0,number,#.###}", f > 0 ? f : 0))
+ MessageFormat.format("{0,number,#.###}", p > 0 ? p : 0),
+ MessageFormat.format("{0,number,#.###}", r > 0 ? r : 0),
+ MessageFormat.format("{0,number,#.###}", f > 0 ? f : 0))
);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/e20dc210/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
index da7013d..7af495c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
@@ -367,22 +367,20 @@ public class LemmatizerFineGrainedReportListener
"% Err", "Precision", "Recall", "F-Measure"));
printLine(tableSize);
- Iterator<String> tagIterator = tags.iterator();
- while (tagIterator.hasNext()) {
- String tag = tagIterator.next();
+ for (String tag : tags) {
int ocurrencies = getTagFrequency(tag);
int errors = getTagErrors(tag);
String rate = MessageFormat.format("{0,number,#.###}",
- (double) errors / ocurrencies);
+ (double) errors / ocurrencies);
double p = getTagPrecision(tag);
double r = getTagRecall(tag);
double f = getTagFMeasure(tag);
printStream.append(String.format(format, tag, errors, ocurrencies, rate,
- MessageFormat.format("{0,number,#.###}", p > 0 ? p : 0),
- MessageFormat.format("{0,number,#.###}", r > 0 ? r : 0),
- MessageFormat.format("{0,number,#.###}", f > 0 ? f : 0))
+ MessageFormat.format("{0,number,#.###}", p > 0 ? p : 0),
+ MessageFormat.format("{0,number,#.###}", r > 0 ? r : 0),
+ MessageFormat.format("{0,number,#.###}", f > 0 ? f : 0))
);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/e20dc210/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
index 369f353..1a40a97 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
@@ -368,9 +368,7 @@ public class POSTaggerFineGrainedReportListener implements
"% Err", "Precision", "Recall", "F-Measure"));
printLine(tableSize);
- Iterator<String> tagIterator = tags.iterator();
- while (tagIterator.hasNext()) {
- String tag = tagIterator.next();
+ for (String tag : tags) {
int ocurrencies = getTagFrequency(tag);
int errors = getTagErrors(tag);
String rate = MessageFormat.format("{0,number,#.###}", (double) errors
[40/50] [abbrv] opennlp git commit: OPENNLP-871: Cleanup for Java 8
Posted by jo...@apache.org.
OPENNLP-871: Cleanup for Java 8
This closes #12
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/bbda5de4
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/bbda5de4
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/bbda5de4
Branch: refs/heads/889
Commit: bbda5de4098e0ffb884384a2fbbccae8b181742b
Parents: dd2aacc
Author: smarthi <sm...@apache.org>
Authored: Wed Dec 21 12:58:26 2016 -0500
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Wed Dec 21 19:17:41 2016 +0100
----------------------------------------------------------------------
.../tools/tokenize/TokenizerCrossValidator.java | 3 +-
.../opennlp/tools/tokenize/TokenizerME.java | 22 +++++-------
.../opennlp/tools/util/ObjectStreamUtils.java | 2 ++
.../tools/tokenize/TokenizerFactoryTest.java | 25 ++++++--------
.../opennlp/uima/chunker/ChunkerTrainer.java | 12 +++----
.../doccat/AbstractDocumentCategorizer.java | 4 +--
.../uima/doccat/DocumentCategorizer.java | 3 +-
.../uima/doccat/DocumentCategorizerTrainer.java | 10 ++----
.../uima/namefind/AbstractNameFinder.java | 6 ++--
.../uima/namefind/NameFinderTrainer.java | 24 ++++----------
.../opennlp/uima/normalizer/Normalizer.java | 5 +--
.../uima/normalizer/StringDictionary.java | 6 ++--
.../main/java/opennlp/uima/parser/Parser.java | 4 +--
.../java/opennlp/uima/postag/POSTagger.java | 4 +--
.../opennlp/uima/postag/POSTaggerTrainer.java | 17 +++++-----
.../sentdetect/SentenceDetectorTrainer.java | 10 +++---
.../opennlp/uima/tokenize/TokenizerTrainer.java | 35 ++++++++++----------
.../opennlp/uima/util/ContainingConstraint.java | 11 ++----
.../java/opennlp/uima/util/OpennlpUtil.java | 4 +--
.../main/java/opennlp/uima/util/UimaUtil.java | 2 +-
20 files changed, 84 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
index 7a85d6a..811165c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
@@ -86,8 +86,7 @@ public class TokenizerCrossValidator {
*/
public void evaluate(ObjectStream<TokenSample> samples, int nFolds) throws IOException {
- CrossValidationPartitioner<TokenSample> partitioner =
- new CrossValidationPartitioner<TokenSample>(samples, nFolds);
+ CrossValidationPartitioner<TokenSample> partitioner = new CrossValidationPartitioner<>(samples, nFolds);
while (partitioner.hasNext()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
index 3d03943..5412c28 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
@@ -18,7 +18,6 @@
package opennlp.tools.tokenize;
import java.io.IOException;
-import java.io.ObjectStreamException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -26,7 +25,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
-
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.ml.EventTrainer;
import opennlp.tools.ml.TrainerFactory;
@@ -127,8 +125,8 @@ public class TokenizerME extends AbstractTokenizer {
this.model = model.getMaxentModel();
this.useAlphaNumericOptimization = factory.isUseAlphaNumericOptmization();
- newTokens = new ArrayList<Span>();
- tokProbs = new ArrayList<Double>(50);
+ newTokens = new ArrayList<>();
+ tokProbs = new ArrayList<>(50);
}
/**
@@ -145,13 +143,13 @@ public class TokenizerME extends AbstractTokenizer {
this.model = model.getMaxentModel();
useAlphaNumericOptimization = model.useAlphaNumericOptimization();
- newTokens = new ArrayList<Span>();
- tokProbs = new ArrayList<Double>(50);
+ newTokens = new ArrayList<>();
+ tokProbs = new ArrayList<>(50);
}
private static Set<String> getAbbreviations(Dictionary abbreviations) {
if(abbreviations == null) {
- return Collections.<String>emptySet();
+ return Collections.emptySet();
}
return abbreviations.asStringSet();
}
@@ -241,7 +239,7 @@ public class TokenizerME extends AbstractTokenizer {
public static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory,
TrainingParameters mlParams) throws IOException {
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
ObjectStream<Event> eventStream = new TokSpanEventStream(samples,
factory.isUseAlphaNumericOptmization(),
@@ -305,7 +303,7 @@ public class TokenizerME extends AbstractTokenizer {
throws IOException {
Factory factory = new Factory();
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
ObjectStream<Event> eventStream = new TokSpanEventStream(samples,
useAlphaNumericOptimization, factory.getAlphanumeric(languageCode),
@@ -334,16 +332,12 @@ public class TokenizerME extends AbstractTokenizer {
* @throws IOException it throws an {@link IOException} if an {@link IOException}
* is thrown during IO operations on a temp file which is
*
- * @throws ObjectStreamException if reading from the {@link ObjectStream} fails
- * created during training.
- *
- *
* @deprecated Use
* {@link #train(ObjectStream, TokenizerFactory, TrainingParameters)}
* and pass in a {@link TokenizerFactory}
*/
public static TokenizerModel train(String languageCode, ObjectStream<TokenSample> samples,
- boolean useAlphaNumericOptimization) throws IOException, ObjectStreamException {
+ boolean useAlphaNumericOptimization) throws IOException {
return train(languageCode, samples, useAlphaNumericOptimization, ModelUtil.createDefaultTrainingParameters());
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-tools/src/main/java/opennlp/tools/util/ObjectStreamUtils.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/ObjectStreamUtils.java b/opennlp-tools/src/main/java/opennlp/tools/util/ObjectStreamUtils.java
index b9b1fe8..9ad98dd 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/ObjectStreamUtils.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/ObjectStreamUtils.java
@@ -31,6 +31,7 @@ public class ObjectStreamUtils {
*
* @return the object stream over the array elements
*/
+ @SafeVarargs
public static <T> ObjectStream<T> createObjectStream(final T... array) {
return new ObjectStream<T>() {
@@ -90,6 +91,7 @@ public class ObjectStreamUtils {
* @param streams
* @return
*/
+ @SafeVarargs
public static <T> ObjectStream<T> createObjectStream(final ObjectStream<T>... streams) {
for (ObjectStream<T> stream : streams) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
index 96d19a6..2fb3d40 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerFactoryTest.java
@@ -18,19 +18,12 @@
package opennlp.tools.tokenize;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Pattern;
-
-import org.junit.Test;
-
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.tokenize.DummyTokenizerFactory.DummyContextGenerator;
@@ -40,6 +33,12 @@ import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
/**
* Tests for the {@link TokenizerFactory} class.
@@ -56,11 +55,10 @@ public class TokenizerFactoryTest {
private static TokenizerModel train(TokenizerFactory factory)
throws IOException {
- return TokenizerME.train(createSampleStream(),
- factory, TrainingParameters.defaultParams());
+ return TokenizerME.train(createSampleStream(), factory, TrainingParameters.defaultParams());
}
- static Dictionary loadAbbDictionary() throws IOException {
+ private static Dictionary loadAbbDictionary() throws IOException {
InputStream in = TokenizerFactoryTest.class.getClassLoader()
.getResourceAsStream("opennlp/tools/sentdetect/abb.xml");
@@ -76,7 +74,7 @@ public class TokenizerFactoryTest {
TokenizerModel model = train(new TokenizerFactory(lang, dic, false, null));
TokenizerFactory factory = model.getFactory();
- assertTrue(factory.getAbbreviationDictionary() instanceof Dictionary);
+ assertTrue(factory.getAbbreviationDictionary() != null);
assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
assertEquals(Factory.DEFAULT_ALPHANUMERIC, factory.getAlphaNumericPattern()
@@ -92,7 +90,7 @@ public class TokenizerFactoryTest {
TokenizerModel fromSerialized = new TokenizerModel(in);
factory = fromSerialized.getFactory();
- assertTrue(factory.getAbbreviationDictionary() instanceof Dictionary);
+ assertTrue(factory.getAbbreviationDictionary() != null);
assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
assertEquals(Factory.DEFAULT_ALPHANUMERIC, factory.getAlphaNumericPattern()
@@ -130,8 +128,7 @@ public class TokenizerFactoryTest {
assertNull(factory.getAbbreviationDictionary());
assertTrue(factory.getContextGenerator() instanceof DefaultTokenContextGenerator);
- assertEquals(Factory.DEFAULT_ALPHANUMERIC, factory.getAlphaNumericPattern()
- .pattern());
+ assertEquals(Factory.DEFAULT_ALPHANUMERIC, factory.getAlphaNumericPattern().pattern());
assertEquals(lang, factory.getLanguageCode());
assertEquals(lang, model.getLanguage());
assertFalse(factory.isUseAlphaNumericOptmization());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
index d22879d..8c6232b 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
@@ -66,7 +66,7 @@ import org.apache.uima.util.ProcessTrace;
*/
public class ChunkerTrainer extends CasConsumer_ImplBase {
- private List<ChunkSample> mChunkSamples = new ArrayList<ChunkSample>();
+ private List<ChunkSample> mChunkSamples = new ArrayList<>();
private UimaContext mContext;
@@ -82,8 +82,6 @@ public class ChunkerTrainer extends CasConsumer_ImplBase {
private Feature mChunkTagFeature;
- private Logger mLogger;
-
private String language;
/**
@@ -95,7 +93,7 @@ public class ChunkerTrainer extends CasConsumer_ImplBase {
mContext = getUimaContext();
- mLogger = mContext.getLogger();
+ Logger mLogger = mContext.getLogger();
if (mLogger.isLoggable(Level.INFO)) {
mLogger.log(Level.INFO, "Initializing the OpenNLP Chunker Trainer.");
@@ -183,9 +181,9 @@ public class ChunkerTrainer extends CasConsumer_ImplBase {
Iterator<AnnotationFS> tokenIterator = tcas.createFilteredIterator(tokenIndex.iterator(),
containingConstraint);
- List<String> tokens = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();;
- List<String> chunkTags = new ArrayList<String>();;
+ List<String> tokens = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
+ List<String> chunkTags = new ArrayList<>();
while (tokenIterator.hasNext()) {
AnnotationFS tokenAnnotation = tokenIterator.next();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java b/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java
index 5abfd76..d2591d5 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/doccat/AbstractDocumentCategorizer.java
@@ -41,8 +41,6 @@ abstract class AbstractDocumentCategorizer extends CasAnnotator_ImplBase {
private UimaContext context;
- private Logger mLogger;
-
private opennlp.tools.doccat.DocumentCategorizer mCategorizer;
private Type mTokenType;
@@ -54,7 +52,7 @@ abstract class AbstractDocumentCategorizer extends CasAnnotator_ImplBase {
this.context = context;
- mLogger = context.getLogger();
+ Logger mLogger = context.getLogger();
if (mLogger.isLoggable(Level.INFO)) {
mLogger.log(Level.INFO, "Initializing the OpenNLP Categorizer.");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
index 5cf337f..cb71a50 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
@@ -56,8 +56,7 @@ public class DocumentCategorizer extends AbstractDocumentCategorizer {
protected void setBestCategory(CAS tcas, String bestCategory) {
FSIndex<AnnotationFS> categoryIndex = tcas.getAnnotationIndex(mCategoryType);
- AnnotationFS categoryAnnotation = categoryIndex.size() > 0 ?
- categoryIndex.iterator().next() : null;
+ AnnotationFS categoryAnnotation;
if (categoryIndex.size() > 0) {
categoryAnnotation = categoryIndex.iterator().next();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
index 31c6392..5b35a82 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
@@ -56,13 +56,9 @@ public class DocumentCategorizerTrainer extends CasConsumer_ImplBase {
private UimaContext mContext;
- private Logger mLogger;
-
private String mModelName;
- private List<DocumentSample> documentSamples = new ArrayList<DocumentSample>();
-
- private Type mTokenType;
+ private List<DocumentSample> documentSamples = new ArrayList<>();
private Type mCategoryType;
@@ -76,7 +72,7 @@ public class DocumentCategorizerTrainer extends CasConsumer_ImplBase {
mContext = getUimaContext();
- mLogger = mContext.getLogger();
+ Logger mLogger = mContext.getLogger();
if (mLogger.isLoggable(Level.INFO)) {
mLogger.log(Level.INFO, "Initializing the OpenNLP Doccat Trainer.");
@@ -95,7 +91,7 @@ public class DocumentCategorizerTrainer extends CasConsumer_ImplBase {
String tokenTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
UimaUtil.SENTENCE_TYPE_PARAMETER);
- mTokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
+ Type mTokenType = CasConsumerUtil.getType(typeSystem, tokenTypeName);
String categoryTypeName = CasConsumerUtil.getRequiredStringParameter(mContext,
"opennlp.uima.doccat.CategoryType");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java b/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
index c0bad5d..dbc604c 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
@@ -123,7 +123,7 @@ abstract class AbstractNameFinder extends CasAnnotator_ImplBase {
final AnnotationComboIterator sentenceNameCombo = new AnnotationComboIterator(cas,
mSentenceType, mNameType);
- List<AnnotationFS> removeAnnotations = new LinkedList<AnnotationFS>();
+ List<AnnotationFS> removeAnnotations = new LinkedList<>();
for (AnnotationIteratorPair annotationIteratorPair : sentenceNameCombo) {
for (AnnotationFS nameAnnotation : annotationIteratorPair.getSubIterator()) {
removeAnnotations.add(nameAnnotation);
@@ -140,9 +140,9 @@ abstract class AbstractNameFinder extends CasAnnotator_ImplBase {
for (AnnotationIteratorPair annotationIteratorPair : sentenceTokenCombo) {
- final List<AnnotationFS> sentenceTokenAnnotationList = new LinkedList<AnnotationFS>();
+ final List<AnnotationFS> sentenceTokenAnnotationList = new LinkedList<>();
- final List<String> sentenceTokenList = new LinkedList<String>();
+ final List<String> sentenceTokenList = new LinkedList<>();
for (AnnotationFS tokenAnnotation : annotationIteratorPair.getSubIterator()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
index dcc0ddc..6207580 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/namefind/NameFinderTrainer.java
@@ -126,7 +126,7 @@ public final class NameFinderTrainer extends CasConsumer_ImplBase {
// - Directly start indexing with a blocking sample stream, the indexer will then write everything
// to disk or could store the events much more space efficient in memory
- private List<NameSample> nameFinderSamples = new ArrayList<NameSample>();
+ private List<NameSample> nameFinderSamples = new ArrayList<>();
private TrainingParameters trainingParams;
/**
@@ -220,7 +220,7 @@ public final class NameFinderTrainer extends CasConsumer_ImplBase {
* @return
*/
private static <T> List<T> iteratorToList(Iterator<T> it) {
- List<T> list = new LinkedList<T>();
+ List<T> list = new LinkedList<>();
while (it.hasNext()) {
list.add(it.next());
@@ -231,19 +231,9 @@ public final class NameFinderTrainer extends CasConsumer_ImplBase {
private static boolean isContaining(AnnotationFS annotation,
AnnotationFS containtedAnnotation) {
- boolean isStartContaining = annotation.getBegin() <= containtedAnnotation
- .getBegin();
- if (!isStartContaining) {
- return false;
- }
-
- boolean isEndContaining = annotation.getEnd() >= containtedAnnotation
- .getEnd();
- if (!isEndContaining) {
- return false;
- }
+ boolean isStartContaining = annotation.getBegin() <= containtedAnnotation.getBegin();
+ return isStartContaining && annotation.getEnd() >= containtedAnnotation.getEnd();
- return true;
}
/**
@@ -258,7 +248,7 @@ public final class NameFinderTrainer extends CasConsumer_ImplBase {
*/
private static Span[] createNames(List<AnnotationFS> tokenList, List<AnnotationFS> entityAnnotations) {
- List<Span> nameList = new LinkedList<Span>();
+ List<Span> nameList = new LinkedList<>();
AnnotationFS currentEntity = null;
@@ -299,7 +289,7 @@ public final class NameFinderTrainer extends CasConsumer_ImplBase {
return nameList.toArray(new Span[nameList.size()]);
}
- /**
+ /*
* Process the given CAS object.
*/
/**
@@ -392,7 +382,7 @@ public final class NameFinderTrainer extends CasConsumer_ImplBase {
if (sampleTraceFile != null) {
samplesOut = new OutputStreamWriter(new FileOutputStream(sampleTraceFile), sampleTraceFileEncoding);
- samples = new SampleTraceStream<NameSample>(samples, samplesOut);
+ samples = new SampleTraceStream<>(samples, samplesOut);
}
Map<String, Object> resourceMap;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java b/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java
index 39150ac..2118a0a 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/normalizer/Normalizer.java
@@ -65,7 +65,7 @@ public class Normalizer extends CasAnnotator_ImplBase {
private static final Set<String> SUPPORTED_TYPES;
static {
- Set<String> supportedTypes = new HashSet<String>();
+ Set<String> supportedTypes = new HashSet<>();
supportedTypes.add(CAS.TYPE_NAME_STRING);
supportedTypes.add(CAS.TYPE_NAME_BYTE);
@@ -232,9 +232,6 @@ public class Normalizer extends CasAnnotator_ImplBase {
.getName())) {
nameAnnotation
.setDoubleValue(mStructureFeature, number.doubleValue());
- } else {
- // assert false : mStructureFeature.getRange().getName()
- // + " is not supported!";
}
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java b/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
index e55f4ae..5631544 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
@@ -34,7 +34,7 @@ import opennlp.tools.util.StringList;
// lookup a string for given token list
public class StringDictionary {
- private Map<StringList, String> entries = new HashMap<StringList, String>();
+ private Map<StringList, String> entries = new HashMap<>();
public StringDictionary() {
}
@@ -44,10 +44,8 @@ public class StringDictionary {
*
* @param in
* @throws IOException
- * @throws InvalidFormatException
*/
- public StringDictionary(InputStream in) throws IOException,
- InvalidFormatException {
+ public StringDictionary(InputStream in) throws IOException {
DictionarySerializer.create(in, new EntryInserter() {
public void insert(Entry entry) throws InvalidFormatException {
String valueString = entry.getAttributes().getValue("value");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java b/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
index a3b3f00..6e7ecda 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
@@ -70,7 +70,7 @@ import org.apache.uima.util.Logger;
public class Parser extends CasAnnotator_ImplBase {
private static class ParseConverter {
- private Map<Integer, Integer> mIndexMap = new HashMap<Integer, Integer>();
+ private Map<Integer, Integer> mIndexMap = new HashMap<>();
private Parse mParseForTagger;
@@ -271,7 +271,7 @@ public class Parser extends CasAnnotator_ImplBase {
Iterator<AnnotationFS> containingTokens = cas.createFilteredIterator(
allTokens.iterator(), containingConstraint);
- List<Span> tokenSpans = new LinkedList<Span>();
+ List<Span> tokenSpans = new LinkedList<>();
while(containingTokens.hasNext()) {
AnnotationFS token = containingTokens.next();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
index 2fdc47c..358e82c 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
@@ -164,9 +164,9 @@ public final class POSTagger extends CasAnnotator_ImplBase {
for (AnnotationIteratorPair annotationIteratorPair : comboIterator) {
- final List<AnnotationFS> sentenceTokenAnnotationList = new LinkedList<AnnotationFS>();
+ final List<AnnotationFS> sentenceTokenAnnotationList = new LinkedList<>();
- final List<String> sentenceTokenList = new LinkedList<String>();
+ final List<String> sentenceTokenList = new LinkedList<>();
for (AnnotationFS tokenAnnotation : annotationIteratorPair.getSubIterator()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
index 23158b9..16e1605 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
@@ -17,26 +17,25 @@
package opennlp.uima.postag;
-import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-
import opennlp.tools.ml.maxent.GIS;
-import opennlp.tools.postag.*;
+import opennlp.tools.postag.POSDictionary;
+import opennlp.tools.postag.POSModel;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.postag.POSTaggerFactory;
+import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.util.ObjectStreamUtils;
import opennlp.tools.util.TrainingParameters;
-import opennlp.tools.util.model.ModelType;
import opennlp.uima.util.AnnotatorUtil;
import opennlp.uima.util.CasConsumerUtil;
import opennlp.uima.util.ContainingConstraint;
import opennlp.uima.util.OpennlpUtil;
import opennlp.uima.util.UimaUtil;
-
import org.apache.uima.UimaContext;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.FSIndex;
@@ -82,7 +81,7 @@ public class POSTaggerTrainer extends CasConsumer_ImplBase {
private Logger mLogger;
- private List<POSSample> mPOSSamples = new ArrayList<POSSample>();
+ private List<POSSample> mPOSSamples = new ArrayList<>();
private String language;
@@ -173,8 +172,8 @@ public class POSTaggerTrainer extends CasConsumer_ImplBase {
ContainingConstraint containingConstraint =
new ContainingConstraint(sentence);
- List<String> tokens = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
+ List<String> tokens = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
Iterator<AnnotationFS> containingTokens = tcas.createFilteredIterator(
allTokens.iterator(), containingConstraint);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/sentdetect/SentenceDetectorTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/sentdetect/SentenceDetectorTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/sentdetect/SentenceDetectorTrainer.java
index 8fa22d7..2c110bd 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/sentdetect/SentenceDetectorTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/sentdetect/SentenceDetectorTrainer.java
@@ -67,7 +67,7 @@ import org.apache.uima.util.ProcessTrace;
*/
public final class SentenceDetectorTrainer extends CasConsumer_ImplBase {
- private List<SentenceSample> sentenceSamples = new ArrayList<SentenceSample>();
+ private List<SentenceSample> sentenceSamples = new ArrayList<>();
private Type mSentenceType;
@@ -75,8 +75,6 @@ public final class SentenceDetectorTrainer extends CasConsumer_ImplBase {
private String language = "en";
- private Logger mLogger;
-
private UimaContext mContext;
private String eosChars;
@@ -94,7 +92,7 @@ public final class SentenceDetectorTrainer extends CasConsumer_ImplBase {
mContext = getUimaContext();
- mLogger = mContext.getLogger();
+ Logger mLogger = mContext.getLogger();
if (mLogger.isLoggable(Level.INFO)) {
mLogger.log(Level.INFO, "Initializing the OpenNLP SentenceDetector " +
@@ -172,11 +170,11 @@ public final class SentenceDetectorTrainer extends CasConsumer_ImplBase {
TrainingParameters mlParams = ModelUtil.createDefaultTrainingParameters();
ObjectStream<SentenceSample> samples = ObjectStreamUtils.createObjectStream(sentenceSamples);
- Writer samplesOut = null;
+ Writer samplesOut;
if (sampleTraceFile != null) {
samplesOut = new OutputStreamWriter(new FileOutputStream(sampleTraceFile), sampleTraceFileEncoding);
- samples = new SampleTraceStream<SentenceSample>(samples, samplesOut);
+ samples = new SampleTraceStream<>(samples, samplesOut);
}
SentenceModel sentenceModel = SentenceDetectorME.train(language, samples,
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
index ece9eca..66d1dfa 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
@@ -29,19 +29,6 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import org.apache.uima.UimaContext;
-import org.apache.uima.cas.CAS;
-import org.apache.uima.cas.FSIndex;
-import org.apache.uima.cas.Type;
-import org.apache.uima.cas.TypeSystem;
-import org.apache.uima.cas.text.AnnotationFS;
-import org.apache.uima.collection.CasConsumer_ImplBase;
-import org.apache.uima.resource.ResourceInitializationException;
-import org.apache.uima.resource.ResourceProcessException;
-import org.apache.uima.util.Level;
-import org.apache.uima.util.Logger;
-import org.apache.uima.util.ProcessTrace;
-
import opennlp.tools.ml.maxent.GIS;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.tokenize.TokenSampleStream;
@@ -58,6 +45,18 @@ import opennlp.uima.util.ContainingConstraint;
import opennlp.uima.util.OpennlpUtil;
import opennlp.uima.util.SampleTraceStream;
import opennlp.uima.util.UimaUtil;
+import org.apache.uima.UimaContext;
+import org.apache.uima.cas.CAS;
+import org.apache.uima.cas.FSIndex;
+import org.apache.uima.cas.Type;
+import org.apache.uima.cas.TypeSystem;
+import org.apache.uima.cas.text.AnnotationFS;
+import org.apache.uima.collection.CasConsumer_ImplBase;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.resource.ResourceProcessException;
+import org.apache.uima.util.Level;
+import org.apache.uima.util.Logger;
+import org.apache.uima.util.ProcessTrace;
/**
* OpenNLP Tokenizer trainer.
@@ -80,10 +79,10 @@ import opennlp.uima.util.UimaUtil;
*/
public final class TokenizerTrainer extends CasConsumer_ImplBase {
- public static final String IS_ALPHA_NUMERIC_OPTIMIZATION =
+ private static final String IS_ALPHA_NUMERIC_OPTIMIZATION =
"opennlp.uima.tokenizer.IsAlphaNumericOptimization";
- private List<TokenSample> tokenSamples = new ArrayList<TokenSample>();
+ private List<TokenSample> tokenSamples = new ArrayList<>();
private UimaContext mContext;
@@ -194,7 +193,7 @@ public final class TokenizerTrainer extends CasConsumer_ImplBase {
Iterator<AnnotationFS> containingTokens = tcas.createFilteredIterator(
allTokens.iterator(), containingConstraint);
- List<Span> openNLPSpans = new LinkedList<Span>();
+ List<Span> openNLPSpans = new LinkedList<>();
while (containingTokens.hasNext()) {
AnnotationFS tokenAnnotation = containingTokens.next();
@@ -231,7 +230,7 @@ public final class TokenizerTrainer extends CasConsumer_ImplBase {
// if trace file
// serialize events ...
- Writer samplesOut = null;
+ Writer samplesOut;
TokenizerModel tokenModel;
if (additionalTrainingDataFile != null) {
@@ -255,7 +254,7 @@ public final class TokenizerTrainer extends CasConsumer_ImplBase {
if (sampleTraceFile != null) {
samplesOut = new OutputStreamWriter(new FileOutputStream(sampleTraceFile), sampleTraceFileEncoding);
- samples = new SampleTraceStream<TokenSample>(samples, samplesOut);
+ samples = new SampleTraceStream<>(samples, samplesOut);
}
tokenModel = TokenizerME.train(language, samples, isSkipAlphaNumerics);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/util/ContainingConstraint.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/util/ContainingConstraint.java b/opennlp-uima/src/main/java/opennlp/uima/util/ContainingConstraint.java
index 20c00ed..57ae612 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/util/ContainingConstraint.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/util/ContainingConstraint.java
@@ -30,8 +30,7 @@ import org.apache.uima.cas.text.AnnotationFS;
public final class ContainingConstraint implements FSMatchConstraint {
private static final long serialVersionUID = 1;
- private Collection<AnnotationFS> mContainingAnnotations =
- new LinkedList<AnnotationFS>();
+ private Collection<AnnotationFS> mContainingAnnotations = new LinkedList<>();
/**
* Initializes a new instance.
@@ -69,12 +68,8 @@ public final class ContainingConstraint implements FSMatchConstraint {
}
private boolean isContaining(AnnotationFS annotation, AnnotationFS containing) {
- if ((containing.getBegin() <= annotation.getBegin())
- && (containing.getEnd() >= annotation.getEnd())) {
- return true;
- } else {
- return false;
- }
+ return (containing.getBegin() <= annotation.getBegin())
+ && (containing.getEnd() >= annotation.getEnd());
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/util/OpennlpUtil.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/util/OpennlpUtil.java b/opennlp-uima/src/main/java/opennlp/uima/util/OpennlpUtil.java
index 59e8566..23d0b3a 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/util/OpennlpUtil.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/util/OpennlpUtil.java
@@ -56,7 +56,7 @@ final public class OpennlpUtil {
}
}
- public static final byte[] loadBytes(File inFile) throws IOException {
+ public static byte[] loadBytes(File inFile) throws IOException {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (InputStream in = new FileInputStream(inFile)) {
@@ -71,7 +71,7 @@ final public class OpennlpUtil {
return bytes.toByteArray();
}
- public static final TrainingParameters loadTrainingParams(String inFileValue,
+ public static TrainingParameters loadTrainingParams(String inFileValue,
boolean isSequenceTrainingAllowed) throws ResourceInitializationException {
TrainingParameters params;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbda5de4/opennlp-uima/src/main/java/opennlp/uima/util/UimaUtil.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/util/UimaUtil.java b/opennlp-uima/src/main/java/opennlp/uima/util/UimaUtil.java
index 610cdfd..5d4efc6 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/util/UimaUtil.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/util/UimaUtil.java
@@ -102,7 +102,7 @@ public final class UimaUtil {
Iterator<AnnotationFS> containingTokens = cas.createFilteredIterator(
allRemoveAnnotations.iterator(), containingConstraint);
- Collection<AnnotationFS> removeAnnotations = new LinkedList<AnnotationFS>();
+ Collection<AnnotationFS> removeAnnotations = new LinkedList<>();
while (containingTokens.hasNext()) {
removeAnnotations.add(containingTokens.next());
[30/50] [abbrv] opennlp git commit: OPENNLP-871: Cleanup code base
for release
Posted by jo...@apache.org.
OPENNLP-871: Cleanup code base for release
This closes #11
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/afc6b65b
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/afc6b65b
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/afc6b65b
Branch: refs/heads/889
Commit: afc6b65b74a4d774cf122fd73733527b181d7d85
Parents: 664d803
Author: smarthi <sm...@apache.org>
Authored: Tue Dec 20 01:47:04 2016 -0500
Committer: Joern Kottmann <jo...@apache.org>
Committed: Tue Dec 20 11:09:13 2016 +0100
----------------------------------------------------------------------
.../builder/MorfologikDictionayBuilder.java | 4 +-
.../builder/XMLDictionaryToTableTool.java | 8 ++--
.../lemmatizer/MorfologikLemmatizer.java | 25 +++++++------
.../tagdict/MorfologikPOSTaggerFactory.java | 18 +++------
.../tagdict/MorfologikTagDictionary.java | 2 +-
.../java/opennlp/tools/chunker/ChunkSample.java | 14 +++----
.../chunker/ChunkSampleSequenceStream.java | 2 +-
.../java/opennlp/tools/chunker/Chunker.java | 8 ++--
.../tools/chunker/ChunkerContextGenerator.java | 2 +-
.../tools/chunker/ChunkerCrossValidator.java | 6 +--
.../tools/chunker/ChunkerEventStream.java | 2 +-
.../opennlp/tools/chunker/ChunkerFactory.java | 4 +-
.../java/opennlp/tools/chunker/ChunkerME.java | 6 +--
.../opennlp/tools/cmdline/ArgumentParser.java | 30 +++++++--------
.../main/java/opennlp/tools/cmdline/CLI.java | 4 +-
.../tools/cmdline/DetailedFMeasureListener.java | 8 ++--
.../tools/cmdline/EvaluationErrorPrinter.java | 14 +++----
.../tools/cmdline/GenerateManualTool.java | 2 +-
.../java/opennlp/tools/cmdline/ModelLoader.java | 3 +-
.../tools/cmdline/StreamFactoryRegistry.java | 9 ++---
.../chunker/ChunkerCrossValidatorTool.java | 2 +-
.../cmdline/chunker/ChunkerEvaluatorTool.java | 2 +-
.../tools/cmdline/chunker/ChunkerMETool.java | 2 +-
.../doccat/DoccatFineGrainedReportListener.java | 38 +++++++++----------
.../tools/cmdline/doccat/DoccatTool.java | 2 +-
.../tools/cmdline/doccat/DoccatTrainerTool.java | 3 +-
.../cmdline/entitylinker/EntityLinkerTool.java | 2 +-
.../LemmatizerFineGrainedReportListener.java | 36 +++++++++---------
.../cmdline/lemmatizer/LemmatizerMETool.java | 2 +-
.../cmdline/namefind/TokenNameFinderTool.java | 2 +-
.../tools/cmdline/postag/POSModelLoader.java | 5 +--
.../POSTaggerFineGrainedReportListener.java | 39 ++++++++++----------
.../tools/cmdline/postag/POSTaggerTool.java | 2 +-
.../cmdline/postag/POSTaggerTrainerTool.java | 26 +++++++------
.../cmdline/tokenizer/CommandLineTokenizer.java | 4 +-
.../opennlp/tools/dictionary/Dictionary.java | 11 ++----
.../java/opennlp/tools/dictionary/Index.java | 2 +-
.../tools/dictionary/serializer/Attributes.java | 2 +-
.../serializer/DictionarySerializer.java | 14 +++----
.../doccat/BagOfWordsFeatureGenerator.java | 2 +-
.../tools/doccat/DoccatCrossValidator.java | 2 +-
.../opennlp/tools/doccat/DoccatFactory.java | 3 +-
.../java/opennlp/tools/doccat/DoccatModel.java | 6 +--
.../DocumentCategorizerContextGenerator.java | 2 +-
.../tools/doccat/DocumentCategorizerME.java | 9 ++---
.../opennlp/tools/doccat/DocumentSample.java | 2 +-
.../tools/doccat/NGramFeatureGenerator.java | 2 +-
.../opennlp/tools/entitylinker/BaseLink.java | 10 +----
.../opennlp/tools/entitylinker/LinkedSpan.java | 8 +---
.../formats/BioNLP2004NameSampleStream.java | 6 +--
.../tools/formats/Conll02NameSampleStream.java | 8 ++--
.../tools/formats/Conll03NameSampleStream.java | 11 +++---
.../tools/formats/ConllXPOSSampleStream.java | 4 +-
.../tools/formats/ad/ADChunkSampleStream.java | 6 +--
.../tools/formats/ad/ADNameSampleStream.java | 19 ++++------
.../tools/formats/ad/ADPOSSampleStream.java | 8 ++--
.../formats/ad/ADPOSSampleStreamFactory.java | 4 +-
.../formats/ad/ADSentenceSampleStream.java | 2 +-
.../ad/ADSentenceSampleStreamFactory.java | 5 +--
.../tools/formats/ad/ADSentenceStream.java | 13 ++-----
.../formats/brat/AnnotationConfiguration.java | 7 +---
.../formats/brat/BratAnnotationStream.java | 3 +-
.../tools/formats/brat/BratDocumentStream.java | 4 +-
.../formats/brat/BratNameSampleStream.java | 8 ++--
.../convert/AbstractToSentenceSampleStream.java | 2 +-
.../formats/convert/ParseToPOSSampleStream.java | 4 +-
.../DefaultLemmatizerContextGenerator.java | 6 +--
.../tools/lemmatizer/DictionaryLemmatizer.java | 6 +--
.../opennlp/tools/lemmatizer/LemmaSample.java | 12 +++---
.../lemmatizer/LemmaSampleEventStream.java | 2 +-
.../lemmatizer/LemmaSampleSequenceStream.java | 2 +-
.../tools/lemmatizer/LemmaSampleStream.java | 10 ++---
.../opennlp/tools/lemmatizer/Lemmatizer.java | 2 +-
.../lemmatizer/LemmatizerContextGenerator.java | 2 +-
.../tools/lemmatizer/LemmatizerFactory.java | 4 +-
.../opennlp/tools/lemmatizer/LemmatizerME.java | 6 +--
.../java/opennlp/tools/ml/TrainerFactory.java | 8 ++--
.../opennlp/tools/ml/maxent/DataStream.java | 4 +-
.../main/java/opennlp/tools/ml/maxent/GIS.java | 6 +--
.../tools/ml/maxent/RealBasicEventStream.java | 2 +-
.../ml/maxent/io/BinaryGISModelWriter.java | 2 +-
.../tools/ml/maxent/io/GISModelWriter.java | 9 ++---
.../ml/maxent/io/OldFormatGISModelReader.java | 6 +--
.../ml/maxent/io/PlainTextGISModelWriter.java | 6 +--
.../io/SuffixSensitiveGISModelReader.java | 3 +-
.../quasinewton/ParallelNegLogLikelihood.java | 4 +-
.../opennlp/tools/ml/model/MaxentModel.java | 18 ++++-----
.../tools/ml/model/OnePassDataIndexer.java | 14 +++----
.../main/java/opennlp/tools/ml/model/Prior.java | 6 +--
.../opennlp/tools/ml/model/SequenceStream.java | 2 +-
.../tools/ml/model/TwoPassDataIndexer.java | 15 ++++----
.../tools/ml/naivebayes/LogProbabilities.java | 2 +-
.../tools/ml/naivebayes/NaiveBayesModel.java | 8 ++--
.../ml/naivebayes/NaiveBayesModelWriter.java | 6 +--
.../tools/ml/naivebayes/NaiveBayesTrainer.java | 2 +-
.../PlainTextNaiveBayesModelWriter.java | 6 +--
.../tools/ml/naivebayes/Probabilities.java | 6 +--
.../java/opennlp/tools/namefind/BilouCodec.java | 5 +--
.../java/opennlp/tools/namefind/BioCodec.java | 7 ++--
.../tools/namefind/DictionaryNameFinder.java | 2 +-
.../tools/namefind/DocumentNameFinder.java | 2 +-
.../tools/namefind/NameContextGenerator.java | 6 +--
.../tools/namefind/NameFinderEventStream.java | 7 ++--
.../opennlp/tools/namefind/NameFinderME.java | 13 +++----
.../java/opennlp/tools/namefind/NameSample.java | 8 ++--
.../namefind/NameSampleSequenceStream.java | 6 +--
.../tools/namefind/NameSampleTypeFilter.java | 6 +--
.../tools/namefind/RegexNameFinderFactory.java | 6 +--
.../opennlp/tools/namefind/TokenNameFinder.java | 4 +-
.../namefind/TokenNameFinderCrossValidator.java | 6 +--
.../namefind/TokenNameFinderEvaluator.java | 8 +---
.../tools/namefind/TokenNameFinderModel.java | 19 ++++------
.../java/opennlp/tools/ngram/NGramModel.java | 2 +-
.../tools/parser/AbstractBottomUpParser.java | 10 ++---
.../tools/parser/AbstractParserEventStream.java | 14 +++----
.../opennlp/tools/parser/ChunkSampleStream.java | 8 ++--
.../java/opennlp/tools/parser/GapLabeler.java | 2 +-
.../java/opennlp/tools/parser/HeadRules.java | 4 +-
.../main/java/opennlp/tools/parser/Parse.java | 17 +++++----
.../main/java/opennlp/tools/parser/Parser.java | 4 +-
.../tools/parser/ParserChunkerFactory.java | 2 +-
.../parser/ParserChunkerSequenceValidator.java | 3 +-
.../tools/parser/ParserCrossValidator.java | 6 +--
.../opennlp/tools/parser/ParserEvaluator.java | 4 +-
.../java/opennlp/tools/parser/ParserModel.java | 12 +++---
.../parser/chunking/BuildContextGenerator.java | 2 +-
.../parser/chunking/CheckContextGenerator.java | 2 +-
.../opennlp/tools/parser/chunking/Parser.java | 22 +++++------
.../parser/chunking/ParserEventStream.java | 39 ++++++++++----------
.../opennlp/tools/parser/lang/en/HeadRules.java | 9 ++---
.../parser/lang/es/AncoraSpanishHeadRules.java | 9 ++---
.../postag/DefaultPOSSequenceValidator.java | 6 +--
.../tools/postag/MutableTagDictionary.java | 4 +-
.../tools/postag/POSContextGenerator.java | 2 +-
.../java/opennlp/tools/postag/POSTagger.java | 10 ++---
.../tools/postag/POSTaggerCrossValidator.java | 2 +-
.../opennlp/tools/postag/TagDictionary.java | 2 +-
.../tools/sentdetect/EndOfSentenceScanner.java | 8 ++--
.../tools/sentdetect/SDContextGenerator.java | 2 +-
.../tools/sentdetect/SentenceDetector.java | 4 +-
.../opennlp/tools/stemmer/PorterStemmer.java | 35 ++++++++----------
.../java/opennlp/tools/stemmer/Stemmer.java | 2 +-
.../tools/tokenize/TokenContextGenerator.java | 2 +-
.../java/opennlp/tools/tokenize/Tokenizer.java | 4 +-
.../tools/util/BeamSearchContextGenerator.java | 2 +-
.../src/main/java/opennlp/tools/util/Heap.java | 16 ++++----
.../opennlp/tools/util/TrainingParameters.java | 6 +--
.../opennlp/tools/util/model/BaseModel.java | 18 ++++-----
148 files changed, 494 insertions(+), 621 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/builder/MorfologikDictionayBuilder.java
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/builder/MorfologikDictionayBuilder.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/builder/MorfologikDictionayBuilder.java
index dbbca4d..54cb95c 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/builder/MorfologikDictionayBuilder.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/builder/MorfologikDictionayBuilder.java
@@ -17,12 +17,10 @@
package opennlp.morfologik.builder;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Path;
import java.util.Properties;
-
import morfologik.stemming.DictionaryMetadata;
import morfologik.stemming.EncoderType;
import morfologik.tools.DictCompile;
@@ -90,7 +88,7 @@ public class MorfologikDictionayBuilder {
}
Properties createProperties(Charset encoding, String separator,
- EncoderType encoderType) throws FileNotFoundException, IOException {
+ EncoderType encoderType) throws IOException {
Properties properties = new Properties();
properties.setProperty("fsa.dict.separator", separator);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
index 0e7f2d5..ef6668e 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/builder/XMLDictionaryToTableTool.java
@@ -59,7 +59,7 @@ public class XMLDictionaryToTableTool extends BasicCmdLineTool {
CmdLineUtil.checkInputFile("dictionary input file", dictInFile);
CmdLineUtil.checkOutputFile("dictionary output file", dictOutFile);
- POSDictionary tagDictionary = null;
+ POSDictionary tagDictionary;
try {
tagDictionary = POSDictionary.create(new FileInputStream(dictInFile));
} catch (IOException e) {
@@ -116,12 +116,10 @@ public class XMLDictionaryToTableTool extends BasicCmdLineTool {
}
private String createEntry(String word, String tag) {
-
- String entry = "" + SEPARATOR +// base
+
+ return "" + SEPARATOR +// base
word + SEPARATOR +
tag;
-
- return entry;
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
index 2090ce5..2798e42 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
@@ -24,6 +24,8 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import morfologik.stemming.Dictionary;
@@ -35,19 +37,18 @@ import opennlp.tools.lemmatizer.DictionaryLemmatizer;
public class MorfologikLemmatizer implements DictionaryLemmatizer {
private IStemmer dictLookup;
- public final Set<String> constantTags = new HashSet<String>(Arrays.asList(
- "NNP", "NP00000"));
+ public final Set<String> constantTags = new HashSet<>(Arrays.asList("NNP", "NP00000"));
public MorfologikLemmatizer(Path dictionaryPath) throws IllegalArgumentException,
IOException {
dictLookup = new DictionaryLookup(Dictionary.read(dictionaryPath));
}
- private HashMap<List<String>, String> getLemmaTagsDict(String word) {
+ private Map<List<String>, String> getLemmaTagsDict(String word) {
List<WordData> wdList = dictLookup.lookup(word);
- HashMap<List<String>, String> dictMap = new HashMap<List<String>, String>();
+ Map<List<String>, String> dictMap = new HashMap<>();
for (WordData wd : wdList) {
- List<String> wordLemmaTags = new ArrayList<String>();
+ List<String> wordLemmaTags = new ArrayList<>();
wordLemmaTags.add(word);
wordLemmaTags.add(wd.getTag().toString());
dictMap.put(wordLemmaTags, wd.getStem().toString());
@@ -56,7 +57,7 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
}
private List<String> getDictKeys(String word, String postag) {
- List<String> keys = new ArrayList<String>();
+ List<String> keys = new ArrayList<>();
if (constantTags.contains(postag)) {
keys.addAll(Arrays.asList(word, postag));
} else {
@@ -65,8 +66,8 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
return keys;
}
- private HashMap<List<String>, String> getDictMap(String word, String postag) {
- HashMap<List<String>, String> dictMap = new HashMap<List<String>, String>();
+ private Map<List<String>, String> getDictMap(String word, String postag) {
+ Map<List<String>, String> dictMap;
if (constantTags.contains(postag)) {
dictMap = this.getLemmaTagsDict(word);
@@ -77,16 +78,16 @@ public class MorfologikLemmatizer implements DictionaryLemmatizer {
}
public String lemmatize(String word, String postag) {
- String lemma = null;
+ String lemma;
List<String> keys = this.getDictKeys(word, postag);
- HashMap<List<String>, String> dictMap = this.getDictMap(word, postag);
+ Map<List<String>, String> dictMap = this.getDictMap(word, postag);
// lookup lemma as value of the map
String keyValue = dictMap.get(keys);
if (keyValue != null) {
lemma = keyValue;
- } else if (keyValue == null && constantTags.contains(postag)) {
+ } else if (constantTags.contains(postag)) {
lemma = word;
- } else if (keyValue == null && word.toUpperCase() == word) {
+ } else if (Objects.equals(word.toUpperCase(), word)) {
lemma = word;
} else {
lemma = word.toLowerCase();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
index 93d6c61..9a6abc9 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikPOSTaggerFactory.java
@@ -26,12 +26,10 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Map;
-
import morfologik.stemming.DictionaryMetadata;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.postag.POSTaggerFactory;
import opennlp.tools.postag.TagDictionary;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.model.ArtifactSerializer;
import opennlp.tools.util.model.ModelUtil;
@@ -53,8 +51,7 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
public MorfologikPOSTaggerFactory() {
}
- public TagDictionary createTagDictionary(File dictionary)
- throws InvalidFormatException, FileNotFoundException, IOException {
+ public TagDictionary createTagDictionary(File dictionary) throws IOException {
if(!dictionary.canRead()) {
throw new FileNotFoundException("Could not read dictionary: " + dictionary.getAbsolutePath());
@@ -87,9 +84,9 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
if (artifactProvider != null) {
Object obj = artifactProvider.getArtifact(MORFOLOGIK_POSDICT);
if (obj != null) {
- byte[] data = (byte[]) artifactProvider
+ byte[] data = artifactProvider
.getArtifact(MORFOLOGIK_POSDICT);
- byte[] info = (byte[]) artifactProvider
+ byte[] info = artifactProvider
.getArtifact(MORFOLOGIK_DICT_INFO);
try {
@@ -121,7 +118,7 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
@Override
public TagDictionary createTagDictionary(InputStream in)
- throws InvalidFormatException, IOException {
+ throws IOException {
throw new UnsupportedOperationException(
"Morfologik POS Tagger factory does not support this operation");
}
@@ -129,8 +126,7 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
@Override
@SuppressWarnings("rawtypes")
public Map<String, ArtifactSerializer> createArtifactSerializersMap() {
- Map<String, ArtifactSerializer> serializers = super
- .createArtifactSerializersMap();
+ Map<String, ArtifactSerializer> serializers = super.createArtifactSerializersMap();
serializers.put(MORFOLOGIK_POSDICT_SUF, new ByteArraySerializer());
serializers.put(MORFOLOGIK_DICT_INFO_SUF, new ByteArraySerializer());
@@ -156,9 +152,7 @@ public class MorfologikPOSTaggerFactory extends POSTaggerFactory {
static class ByteArraySerializer implements ArtifactSerializer<byte[]> {
- public byte[] create(InputStream in) throws IOException,
- InvalidFormatException {
-
+ public byte[] create(InputStream in) throws IOException {
return ModelUtil.read(in);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
index b34ca2b..a57b4bb 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/tagdict/MorfologikTagDictionary.java
@@ -77,7 +77,7 @@ public class MorfologikTagDictionary implements TagDictionary {
List<WordData> data = dictLookup.lookup(word);
if (data != null && data.size() > 0) {
- List<String> tags = new ArrayList<String>(data.size());
+ List<String> tags = new ArrayList<>(data.size());
for (int i = 0; i < data.size(); i++) {
tags.add(data.get(i).getTag().toString());
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
index da71d36..c61b051 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
@@ -47,9 +47,9 @@ public class ChunkSample {
validateArguments(sentence.length, tags.length, preds.length);
- this.sentence = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(sentence)));
- this.tags = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(tags)));
- this.preds = Collections.unmodifiableList(new ArrayList<String>(Arrays.asList(preds)));
+ this.sentence = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(sentence)));
+ this.tags = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(tags)));
+ this.preds = Collections.unmodifiableList(new ArrayList<>(Arrays.asList(preds)));
}
/**
@@ -66,9 +66,9 @@ public class ChunkSample {
validateArguments(sentence.size(), tags.size(), preds.size());
- this.sentence = Collections.unmodifiableList(new ArrayList<String>(sentence));
- this.tags = Collections.unmodifiableList(new ArrayList<String>(tags));
- this.preds = Collections.unmodifiableList(new ArrayList<String>(preds));
+ this.sentence = Collections.unmodifiableList(new ArrayList<>(sentence));
+ this.tags = Collections.unmodifiableList(new ArrayList<>(tags));
+ this.preds = Collections.unmodifiableList(new ArrayList<>(preds));
}
/** Gets the training sentence */
@@ -109,7 +109,7 @@ public class ChunkSample {
validateArguments(aSentence.length, aTags.length, aPreds.length);
// initialize with the list maximum size
- List<Span> phrases = new ArrayList<Span>(aSentence.length);
+ List<Span> phrases = new ArrayList<>(aSentence.length);
String startTag = "";
int startIndex = 0;
boolean foundPhrase = false;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
index e4da42f..8935829 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSampleSequenceStream.java
@@ -53,7 +53,7 @@ public class ChunkSampleSequenceStream implements SequenceStream {
events[i] = new Event(tags[i], context);
}
- return new Sequence<ChunkSample>(events,sample);
+ return new Sequence<>(events,sample);
}
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
index b26f01b..c496d12 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
@@ -33,7 +33,7 @@ public interface Chunker {
*
* @return an array of chunk tags for each token in the sequence.
*/
- public String[] chunk(String[] toks, String tags[]);
+ String[] chunk(String[] toks, String tags[]);
/**
* Generates tagged chunk spans for the given sequence returning the result in a span array.
@@ -43,7 +43,7 @@ public interface Chunker {
*
* @return an array of spans with chunk tags for each chunk in the sequence.
*/
- public Span[] chunkAsSpans(String[] toks, String tags[]);
+ Span[] chunkAsSpans(String[] toks, String tags[]);
/**
* Returns the top k chunk sequences for the specified sentence with the specified pos-tags
@@ -52,7 +52,7 @@ public interface Chunker {
*
* @return the top k chunk sequences for the specified sentence.
*/
- public Sequence[] topKSequences(String[] sentence, String[] tags);
+ Sequence[] topKSequences(String[] sentence, String[] tags);
/**
* Returns the top k chunk sequences for the specified sentence with the specified pos-tags
@@ -62,5 +62,5 @@ public interface Chunker {
*
* @return the top k chunk sequences for the specified sentence.
*/
- public Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore);
+ Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerContextGenerator.java
index 8b1e745..25e531a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerContextGenerator.java
@@ -32,5 +32,5 @@ public interface ChunkerContextGenerator extends BeamSearchContextGenerator<Stri
* @param preds The previous decisions made in the taging of this sequence. Only indices less than i will be examined.
* @return An array of predictive contexts on which a model basis its decisions.
*/
- public String[] getContext(int i, String[] toks, String[] tags, String[] preds);
+ String[] getContext(int i, String[] toks, String[] tags, String[] preds);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
index e9ce18d..f696503 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
@@ -18,8 +18,6 @@
package opennlp.tools.chunker;
import java.io.IOException;
-
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
@@ -53,8 +51,8 @@ public class ChunkerCrossValidator {
* @throws IOException
*/
public void evaluate(ObjectStream<ChunkSample> samples, int nFolds)
- throws IOException, InvalidFormatException {
- CrossValidationPartitioner<ChunkSample> partitioner = new CrossValidationPartitioner<ChunkSample>(
+ throws IOException {
+ CrossValidationPartitioner<ChunkSample> partitioner = new CrossValidationPartitioner<>(
samples, nFolds);
while (partitioner.hasNext()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
index 118357c..d72d115 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
@@ -47,7 +47,7 @@ public class ChunkerEventStream extends AbstractEventStream<ChunkSample> {
protected Iterator<Event> createEvents(ChunkSample sample) {
if (sample != null) {
- List<Event> events = new ArrayList<Event>();
+ List<Event> events = new ArrayList<>();
String[] toksArray = sample.getSentence();
String[] tagsArray = sample.getTags();
String[] predsArray = sample.getPreds();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerFactory.java
index c59be29..1cb772f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerFactory.java
@@ -38,9 +38,7 @@ public class ChunkerFactory extends BaseToolFactory {
return new ChunkerFactory();
}
try {
- ChunkerFactory theFactory = ExtensionLoader.instantiateExtension(
- ChunkerFactory.class, subclassName);
- return theFactory;
+ return ExtensionLoader.instantiateExtension(ChunkerFactory.class, subclassName);
} catch (Exception e) {
String msg = "Could not instantiate the " + subclassName
+ ". The initialization throw an exception.";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
index e5b1073..3ed4f9c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
@@ -77,7 +77,7 @@ public class ChunkerME implements Chunker {
this.model = model.getChunkerSequenceModel();
}
else {
- this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+ this.model = new opennlp.tools.ml.BeamSearch<>(beamSize,
model.getChunkerModel(), 0);
}
}
@@ -101,7 +101,7 @@ public class ChunkerME implements Chunker {
this.model = model.getChunkerSequenceModel();
}
else {
- this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+ this.model = new opennlp.tools.ml.BeamSearch<>(beamSize,
model.getChunkerModel(), 0);
}
}
@@ -168,7 +168,7 @@ public class ChunkerME implements Chunker {
beamSize = Integer.parseInt(beamSizeString);
}
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
TrainerType trainerType = TrainerFactory.getTrainerType(mlParams.getSettings());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/ArgumentParser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/ArgumentParser.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/ArgumentParser.java
index cfc17aa..ca9a650 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/ArgumentParser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/ArgumentParser.java
@@ -51,19 +51,19 @@ public class ArgumentParser {
public @Retention(RetentionPolicy.RUNTIME)
@interface OptionalParameter {
- public static final String DEFAULT_CHARSET = "DEFAULT_CHARSET";
- public String defaultValue() default "";
+ String DEFAULT_CHARSET = "DEFAULT_CHARSET";
+ String defaultValue() default "";
}
public @Retention(RetentionPolicy.RUNTIME)
@interface ParameterDescription {
- public String valueName();
- public String description() default "";
+ String valueName();
+ String description() default "";
}
private interface ArgumentFactory {
- static final String INVALID_ARG = "Invalid argument: %s %s \n";
+ String INVALID_ARG = "Invalid argument: %s %s \n";
Object parseArgument(Method method, String argName, String argValue);
}
@@ -148,7 +148,7 @@ public class ArgumentParser {
private static final Map<Class<?>, ArgumentFactory> argumentFactories;
static {
- Map<Class<?>, ArgumentFactory> factories = new HashMap<Class<?>, ArgumentParser.ArgumentFactory>();
+ Map<Class<?>, ArgumentFactory> factories = new HashMap<>();
factories.put(Integer.class, new IntegerArgumentFactory());
factories.put(Boolean.class, new BooleanArgumentFactory());
factories.put(String.class, new StringArgumentFactory());
@@ -205,9 +205,7 @@ public class ArgumentParser {
// name length is checked to be at least 4 prior
parameterNameChars[3] = Character.toLowerCase(parameterNameChars[3]);
- String parameterName = "-" + new String(parameterNameChars).substring(3);
-
- return parameterName;
+ return "-" + new String(parameterNameChars).substring(3);
}
/**
@@ -270,9 +268,9 @@ public class ArgumentParser {
public static <T> List<Argument> createArguments(Class<T>... argProxyInterfaces) {
checkProxyInterfaces(argProxyInterfaces);
- Set<String> duplicateFilter = new HashSet<String>();
+ Set<String> duplicateFilter = new HashSet<>();
- List<Argument> arguments = new LinkedList<Argument>();
+ List<Argument> arguments = new LinkedList<>();
for (Class<T> argProxyInterface : argProxyInterfaces) {
if (null != argProxyInterface) {
@@ -321,7 +319,7 @@ public class ArgumentParser {
public static <T> String createUsage(Class<T>... argProxyInterfaces) {
checkProxyInterfaces(argProxyInterfaces);
- Set<String> duplicateFilter = new HashSet<String>();
+ Set<String> duplicateFilter = new HashSet<>();
StringBuilder usage = new StringBuilder();
StringBuilder details = new StringBuilder();
@@ -348,7 +346,7 @@ public class ArgumentParser {
usage.append(paramName).append(' ').append(desc.valueName());
details.append('\t').append(paramName).append(' ').append(desc.valueName()).append('\n');
- if(desc.description() != null && desc.description().length() > 0) {
+ if(desc.description().length() > 0) {
details.append("\t\t").append(desc.description()).append('\n');
}
@@ -425,7 +423,7 @@ public class ArgumentParser {
}
int argumentCount = 0;
- List<String> parameters = new ArrayList<String>(Arrays.asList(args));
+ List<String> parameters = new ArrayList<>(Arrays.asList(args));
for (Class<T> argProxyInterface : argProxyInterfaces) {
for (Method method : argProxyInterface.getMethods()) {
@@ -482,7 +480,7 @@ public class ArgumentParser {
if (!validateArguments(args, argProxyInterface))
throw new IllegalArgumentException("Passed args must be valid!");
- Map<String, Object> arguments = new HashMap<String, Object>();
+ Map<String, Object> arguments = new HashMap<>();
for (Method method : argProxyInterface.getMethods()) {
@@ -531,7 +529,7 @@ public class ArgumentParser {
* @return arguments pertaining to argProxyInterface
*/
public static <T> String[] filter(String args[], Class<T> argProxyInterface) {
- ArrayList<String> parameters = new ArrayList<String>(args.length);
+ ArrayList<String> parameters = new ArrayList<>(args.length);
for (Method method : argProxyInterface.getMethods()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/CLI.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/CLI.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/CLI.java
index 6cfe0b7..7abb6d4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/CLI.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/CLI.java
@@ -78,9 +78,9 @@ public final class CLI {
private static Map<String, CmdLineTool> toolLookupMap;
static {
- toolLookupMap = new LinkedHashMap<String, CmdLineTool>();
+ toolLookupMap = new LinkedHashMap<>();
- List<CmdLineTool> tools = new LinkedList<CmdLineTool>();
+ List<CmdLineTool> tools = new LinkedList<>();
// Document Categorizer
tools.add(new DoccatTool());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
index ccf6884..4e3ec88 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
@@ -41,7 +41,7 @@ public abstract class DetailedFMeasureListener<T> implements
private int samples = 0;
private Stats generalStats = new Stats();
- private Map<String, Stats> statsForOutcome = new HashMap<String, Stats>();
+ private Map<String, Stats> statsForOutcome = new HashMap<>();
protected abstract Span[] asSpanArray(T sample);
@@ -59,8 +59,8 @@ public abstract class DetailedFMeasureListener<T> implements
Span[] references = asSpanArray(reference);
Span[] predictions = asSpanArray(prediction);
- Set<Span> refSet = new HashSet<Span>(Arrays.asList(references));
- Set<Span> predSet = new HashSet<Span>(Arrays.asList(predictions));
+ Set<Span> refSet = new HashSet<>(Arrays.asList(references));
+ Set<Span> predSet = new HashSet<>(Arrays.asList(predictions));
for (Span ref : refSet) {
if (predSet.contains(ref)) {
@@ -129,7 +129,7 @@ public abstract class DetailedFMeasureListener<T> implements
zeroOrPositive(generalStats.getRecallScore() * 100),
zeroOrPositive(generalStats.getFMeasure() * 100)));
ret.append("\n");
- SortedSet<String> set = new TreeSet<String>(new F1Comparator());
+ SortedSet<String> set = new TreeSet<>(new F1Comparator());
set.addAll(statsForOutcome.keySet());
for (String type : set) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java
index 6fda10b..51e1161 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java
@@ -40,8 +40,8 @@ public abstract class EvaluationErrorPrinter<T> implements EvaluationMonitor<T>
// for the sentence detector
protected void printError(Span references[], Span predictions[],
T referenceSample, T predictedSample, String sentence) {
- List<Span> falseNegatives = new ArrayList<Span>();
- List<Span> falsePositives = new ArrayList<Span>();
+ List<Span> falseNegatives = new ArrayList<>();
+ List<Span> falsePositives = new ArrayList<>();
findErrors(references, predictions, falseNegatives, falsePositives);
@@ -57,8 +57,8 @@ public abstract class EvaluationErrorPrinter<T> implements EvaluationMonitor<T>
// for namefinder, chunker...
protected void printError(String id, Span references[], Span predictions[],
T referenceSample, T predictedSample, String[] sentenceTokens) {
- List<Span> falseNegatives = new ArrayList<Span>();
- List<Span> falsePositives = new ArrayList<Span>();
+ List<Span> falseNegatives = new ArrayList<>();
+ List<Span> falsePositives = new ArrayList<>();
findErrors(references, predictions, falseNegatives, falsePositives);
@@ -83,9 +83,9 @@ public abstract class EvaluationErrorPrinter<T> implements EvaluationMonitor<T>
// for pos tagger
protected void printError(String references[], String predictions[],
T referenceSample, T predictedSample, String[] sentenceTokens) {
- List<String> filteredDoc = new ArrayList<String>();
- List<String> filteredRefs = new ArrayList<String>();
- List<String> filteredPreds = new ArrayList<String>();
+ List<String> filteredDoc = new ArrayList<>();
+ List<String> filteredRefs = new ArrayList<>();
+ List<String> filteredPreds = new ArrayList<>();
for (int i = 0; i < references.length; i++) {
if (!references[i].equals(predictions[i])) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
index b2a9c42..00074ea 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
@@ -45,7 +45,7 @@ public class GenerateManualTool {
appendHeader(sb);
// organize by package name
- LinkedHashMap<String, Map<String, CmdLineTool>> packageNameToolMap = new LinkedHashMap<String, Map<String, CmdLineTool>>();
+ LinkedHashMap<String, Map<String, CmdLineTool>> packageNameToolMap = new LinkedHashMap<>();
for (String toolName : CLI.getToolLookupMap().keySet()) {
CmdLineTool tool = CLI.getToolLookupMap().get(toolName);
String packageName = tool.getClass().getPackage().getName();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/ModelLoader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/ModelLoader.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/ModelLoader.java
index 3e5db54..561a6f8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/ModelLoader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/ModelLoader.java
@@ -43,8 +43,7 @@ public abstract class ModelLoader<T> {
this.modelName = modelName;
}
- protected abstract T loadModel(InputStream modelIn) throws
- IOException, InvalidFormatException;
+ protected abstract T loadModel(InputStream modelIn) throws IOException;
public T load(File modelFile) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/StreamFactoryRegistry.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/StreamFactoryRegistry.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/StreamFactoryRegistry.java
index 9173c22..2573e2e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/StreamFactoryRegistry.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/StreamFactoryRegistry.java
@@ -60,8 +60,7 @@ import opennlp.tools.formats.ontonotes.OntoNotesParseSampleStreamFactory;
*/
public final class StreamFactoryRegistry {
- private static final Map<Class, Map<String, ObjectStreamFactory>> registry =
- new HashMap<Class, Map<String, ObjectStreamFactory>>();
+ private static final Map<Class, Map<String, ObjectStreamFactory>> registry = new HashMap<>();
static {
ChunkerSampleStreamFactory.registerFactory();
@@ -129,7 +128,7 @@ public final class StreamFactoryRegistry {
boolean result;
Map<String, ObjectStreamFactory> formats = registry.get(sampleClass);
if (null == formats) {
- formats = new HashMap<String, ObjectStreamFactory>();
+ formats = new HashMap<>();
}
if (!formats.containsKey(formatName)) {
formats.put(formatName, factory);
@@ -198,10 +197,8 @@ public final class StreamFactoryRegistry {
try {
return (ObjectStreamFactory<T>) factoryClazz.newInstance();
- } catch (InstantiationException e) {
+ } catch (InstantiationException | IllegalAccessException e) {
return null;
- } catch (IllegalAccessException e) {
- return null;
}
} catch (ClassNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
index a4e0a4c..2724d88 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
@@ -57,7 +57,7 @@ public final class ChunkerCrossValidatorTool
mlParams = ModelUtil.createDefaultTrainingParameters();
}
- List<EvaluationMonitor<ChunkSample>> listeners = new LinkedList<EvaluationMonitor<ChunkSample>>();
+ List<EvaluationMonitor<ChunkSample>> listeners = new LinkedList<>();
ChunkerDetailedFMeasureListener detailedFMeasureListener = null;
if (params.getMisclassified()) {
listeners.add(new ChunkEvaluationErrorListener());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
index 58d8703..01324cc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
@@ -54,7 +54,7 @@ public final class ChunkerEvaluatorTool
ChunkerModel model = new ChunkerModelLoader().load(params.getModel());
- List<EvaluationMonitor<ChunkSample>> listeners = new LinkedList<EvaluationMonitor<ChunkSample>>();
+ List<EvaluationMonitor<ChunkSample>> listeners = new LinkedList<>();
ChunkerDetailedFMeasureListener detailedFMeasureListener = null;
if(params.getMisclassified()) {
listeners.add(new ChunkEvaluationErrorListener());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
index 25c4465..5b87c9e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
@@ -50,7 +50,7 @@ public class ChunkerMETool extends BasicCmdLineTool {
ChunkerME chunker = new ChunkerME(model);
- ObjectStream<String> lineStream = null;
+ ObjectStream<String> lineStream;
PerformanceMonitor perfMon = null;
try {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
index e07dc4f..0652d0b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
@@ -26,14 +26,12 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-
import opennlp.tools.doccat.DoccatEvaluationMonitor;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.util.Span;
@@ -330,7 +328,7 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
printStream.append("\n");
}
- private static final String generateAlphaLabel(int index) {
+ private static String generateAlphaLabel(int index) {
char labelChars[] = new char[3];
int i;
@@ -355,20 +353,20 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
private int maximumSentenceLength = Integer.MIN_VALUE;
// token statistics
- private final Map<String, Mean> tokAccuracies = new HashMap<String, Mean>();
- private final Map<String, Counter> tokOcurrencies = new HashMap<String, Counter>();
- private final Map<String, Counter> tokErrors = new HashMap<String, Counter>();
+ private final Map<String, Mean> tokAccuracies = new HashMap<>();
+ private final Map<String, Counter> tokOcurrencies = new HashMap<>();
+ private final Map<String, Counter> tokErrors = new HashMap<>();
// tag statistics
- private final Map<String, Counter> tagOcurrencies = new HashMap<String, Counter>();
- private final Map<String, Counter> tagErrors = new HashMap<String, Counter>();
- private final Map<String, FMeasure> tagFMeasure = new HashMap<String, FMeasure>();
+ private final Map<String, Counter> tagOcurrencies = new HashMap<>();
+ private final Map<String, Counter> tagErrors = new HashMap<>();
+ private final Map<String, FMeasure> tagFMeasure = new HashMap<>();
// represents a Confusion Matrix that aggregates all tokens
- private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<String, ConfusionMatrixLine>();
+ private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<>();
// represents a set of Confusion Matrix for each token
- private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<String, Map<String, ConfusionMatrixLine>>();
+ private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<>();
public void add(DocumentSample reference, DocumentSample prediction) {
int length = reference.getText().length;
@@ -447,13 +445,13 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
private void updateTagFMeasure(String[] refs, String[] preds) {
// create a set with all tags
- Set<String> tags = new HashSet<String>(Arrays.asList(refs));
+ Set<String> tags = new HashSet<>(Arrays.asList(refs));
tags.addAll(Arrays.asList(preds));
// create samples for each tag
for (String tag : tags) {
- List<Span> reference = new ArrayList<Span>();
- List<Span> prediction = new ArrayList<Span>();
+ List<Span> reference = new ArrayList<>();
+ List<Span> prediction = new ArrayList<>();
for (int i = 0; i < refs.length; i++) {
if (refs[i].equals(tag)) {
reference.add(new Span(i, i + 1));
@@ -509,7 +507,7 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
}
public SortedSet<String> getTokensOrderedByFrequency() {
- SortedSet<String> toks = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> toks = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -532,7 +530,7 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
}
public SortedSet<String> getTokensOrderedByNumberOfErrors() {
- SortedSet<String> toks = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> toks = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -573,7 +571,7 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
}
public SortedSet<String> getTagsOrderedByErrors() {
- SortedSet<String> tags = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> tags = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -638,9 +636,9 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
private SortedSet<String> getConfusionMatrixTagset(
Map<String, ConfusionMatrixLine> data) {
- SortedSet<String> tags = new TreeSet<String>(new CategoryComparator(data));
+ SortedSet<String> tags = new TreeSet<>(new CategoryComparator(data));
tags.addAll(data.keySet());
- List<String> col = new LinkedList<String>();
+ List<String> col = new LinkedList<>();
for (String t : tags) {
col.addAll(data.get(t).line.keySet());
}
@@ -693,7 +691,7 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
*/
private static class ConfusionMatrixLine {
- private Map<String, Counter> line = new HashMap<String, Counter>();
+ private Map<String, Counter> line = new HashMap<>();
private String ref;
private int total = 0;
private int correct = 0;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTool.java
index d41dabb..dbc6e6b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTool.java
@@ -54,7 +54,7 @@ public class DoccatTool extends BasicCmdLineTool {
DocumentCategorizerME doccat = new DocumentCategorizerME(model);
- /**
+ /*
* moved initialization to the try block to catch new IOException
*/
ObjectStream<String> documentStream;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTrainerTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTrainerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTrainerTool.java
index 421c57f..91ee019 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTrainerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatTrainerTool.java
@@ -99,8 +99,7 @@ public class DoccatTrainerTool
static FeatureGenerator[] createFeatureGenerators(String featureGeneratorsNames) {
if(featureGeneratorsNames == null) {
- FeatureGenerator[] def = {new BagOfWordsFeatureGenerator()};
- return def;
+ return new FeatureGenerator[]{new BagOfWordsFeatureGenerator()};
}
String[] classes = featureGeneratorsNames.split(",");
FeatureGenerator[] featureGenerators = new FeatureGenerator[classes.length];
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/entitylinker/EntityLinkerTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/entitylinker/EntityLinkerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/entitylinker/EntityLinkerTool.java
index dbdb27a..f82e362 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/entitylinker/EntityLinkerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/entitylinker/EntityLinkerTool.java
@@ -84,7 +84,7 @@ public class EntityLinkerTool extends BasicCmdLineTool {
ObjectStream<String> untokenizedLineStream = new PlainTextByLineStream(
new SystemInputStreamFactory(), SystemInputStreamFactory.encoding());
- List<NameSample> document = new ArrayList<NameSample>();
+ List<NameSample> document = new ArrayList<>();
String line;
while ((line = untokenizedLineStream.read()) != null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
index 7af495c..b72e889 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
@@ -459,7 +459,7 @@ public class LemmatizerFineGrainedReportListener
printStream.append("\n");
}
- private static final String generateAlphaLabel(int index) {
+ private static String generateAlphaLabel(int index) {
char labelChars[] = new char[3];
int i;
@@ -484,20 +484,20 @@ public class LemmatizerFineGrainedReportListener
private int maximumSentenceLength = Integer.MIN_VALUE;
// token statistics
- private final Map<String, Mean> tokAccuracies = new HashMap<String, Mean>();
- private final Map<String, Counter> tokOcurrencies = new HashMap<String, Counter>();
- private final Map<String, Counter> tokErrors = new HashMap<String, Counter>();
+ private final Map<String, Mean> tokAccuracies = new HashMap<>();
+ private final Map<String, Counter> tokOcurrencies = new HashMap<>();
+ private final Map<String, Counter> tokErrors = new HashMap<>();
// tag statistics
- private final Map<String, Counter> tagOcurrencies = new HashMap<String, Counter>();
- private final Map<String, Counter> tagErrors = new HashMap<String, Counter>();
- private final Map<String, FMeasure> tagFMeasure = new HashMap<String, FMeasure>();
+ private final Map<String, Counter> tagOcurrencies = new HashMap<>();
+ private final Map<String, Counter> tagErrors = new HashMap<>();
+ private final Map<String, FMeasure> tagFMeasure = new HashMap<>();
// represents a Confusion Matrix that aggregates all tokens
- private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<String, ConfusionMatrixLine>();
+ private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<>();
// represents a set of Confusion Matrix for each token
- private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<String, Map<String, ConfusionMatrixLine>>();
+ private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<>();
public void add(LemmaSample reference, LemmaSample prediction) {
int length = reference.getTokens().length;
@@ -576,13 +576,13 @@ public class LemmatizerFineGrainedReportListener
private void updateTagFMeasure(String[] refs, String[] preds) {
// create a set with all tags
- Set<String> tags = new HashSet<String>(Arrays.asList(refs));
+ Set<String> tags = new HashSet<>(Arrays.asList(refs));
tags.addAll(Arrays.asList(preds));
// create samples for each tag
for (String tag : tags) {
- List<Span> reference = new ArrayList<Span>();
- List<Span> prediction = new ArrayList<Span>();
+ List<Span> reference = new ArrayList<>();
+ List<Span> prediction = new ArrayList<>();
for (int i = 0; i < refs.length; i++) {
if (refs[i].equals(tag)) {
reference.add(new Span(i, i + 1));
@@ -638,7 +638,7 @@ public class LemmatizerFineGrainedReportListener
}
public SortedSet<String> getTokensOrderedByFrequency() {
- SortedSet<String> toks = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> toks = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -661,7 +661,7 @@ public class LemmatizerFineGrainedReportListener
}
public SortedSet<String> getTokensOrderedByNumberOfErrors() {
- SortedSet<String> toks = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> toks = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -702,7 +702,7 @@ public class LemmatizerFineGrainedReportListener
}
public SortedSet<String> getTagsOrderedByErrors() {
- SortedSet<String> tags = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> tags = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -767,10 +767,10 @@ public class LemmatizerFineGrainedReportListener
private SortedSet<String> getConfusionMatrixTagset(
Map<String, ConfusionMatrixLine> data) {
- SortedSet<String> tags = new TreeSet<String>(
+ SortedSet<String> tags = new TreeSet<>(
new CategoryComparator(data));
tags.addAll(data.keySet());
- List<String> col = new LinkedList<String>();
+ List<String> col = new LinkedList<>();
for (String t : tags) {
col.addAll(data.get(t).line.keySet());
}
@@ -824,7 +824,7 @@ public class LemmatizerFineGrainedReportListener
*/
private static class ConfusionMatrixLine {
- private Map<String, Counter> line = new HashMap<String, Counter>();
+ private Map<String, Counter> line = new HashMap<>();
private String ref;
private int total = 0;
private int correct = 0;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerMETool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerMETool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerMETool.java
index 8805bca..966111a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerMETool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerMETool.java
@@ -51,7 +51,7 @@ public class LemmatizerMETool extends BasicCmdLineTool {
LemmatizerME lemmatizer = new LemmatizerME(model);
- ObjectStream<String> lineStream = null;
+ ObjectStream<String> lineStream;
PerformanceMonitor perfMon = null;
try {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTool.java
index 7f05a78..6e8b86f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTool.java
@@ -81,7 +81,7 @@ public final class TokenNameFinderTool extends BasicCmdLineTool {
}
}
- List<Span> names = new ArrayList<Span>();
+ List<Span> names = new ArrayList<>();
for (TokenNameFinder nameFinder : nameFinders) {
Collections.addAll(names, nameFinder.find(whitespaceTokenizerLine));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSModelLoader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSModelLoader.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSModelLoader.java
index 2bdfe7b..40632cf 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSModelLoader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSModelLoader.java
@@ -19,10 +19,8 @@ package opennlp.tools.cmdline.postag;
import java.io.IOException;
import java.io.InputStream;
-
import opennlp.tools.cmdline.ModelLoader;
import opennlp.tools.postag.POSModel;
-import opennlp.tools.util.InvalidFormatException;
/**
* Loads a POS Tagger Model for the command line tools.
@@ -36,8 +34,7 @@ public final class POSModelLoader extends ModelLoader<POSModel>{
}
@Override
- protected POSModel loadModel(InputStream modelIn) throws IOException,
- InvalidFormatException {
+ protected POSModel loadModel(InputStream modelIn) throws IOException {
return new POSModel(modelIn);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
index 1a40a97..4149c97 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
@@ -466,7 +466,7 @@ public class POSTaggerFineGrainedReportListener implements
printStream.append("\n");
}
- private static final String generateAlphaLabel(int index) {
+ private static String generateAlphaLabel(int index) {
char labelChars[] = new char[3];
int i;
@@ -491,20 +491,20 @@ public class POSTaggerFineGrainedReportListener implements
private int maximumSentenceLength = Integer.MIN_VALUE;
// token statistics
- private final Map<String, Mean> tokAccuracies = new HashMap<String, Mean>();
- private final Map<String, Counter> tokOcurrencies = new HashMap<String, Counter>();
- private final Map<String, Counter> tokErrors = new HashMap<String, Counter>();
+ private final Map<String, Mean> tokAccuracies = new HashMap<>();
+ private final Map<String, Counter> tokOcurrencies = new HashMap<>();
+ private final Map<String, Counter> tokErrors = new HashMap<>();
// tag statistics
- private final Map<String, Counter> tagOcurrencies = new HashMap<String, Counter>();
- private final Map<String, Counter> tagErrors = new HashMap<String, Counter>();
- private final Map<String, FMeasure> tagFMeasure = new HashMap<String, FMeasure>();
+ private final Map<String, Counter> tagOcurrencies = new HashMap<>();
+ private final Map<String, Counter> tagErrors = new HashMap<>();
+ private final Map<String, FMeasure> tagFMeasure = new HashMap<>();
// represents a Confusion Matrix that aggregates all tokens
- private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<String, ConfusionMatrixLine>();
+ private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<>();
// represents a set of Confusion Matrix for each token
- private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<String, Map<String, ConfusionMatrixLine>>();
+ private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<>();
public void add(POSSample reference, POSSample prediction) {
int length = reference.getSentence().length;
@@ -583,13 +583,13 @@ public class POSTaggerFineGrainedReportListener implements
private void updateTagFMeasure(String[] refs, String[] preds) {
// create a set with all tags
- Set<String> tags = new HashSet<String>(Arrays.asList(refs));
+ Set<String> tags = new HashSet<>(Arrays.asList(refs));
tags.addAll(Arrays.asList(preds));
// create samples for each tag
for (String tag : tags) {
- List<Span> reference = new ArrayList<Span>();
- List<Span> prediction = new ArrayList<Span>();
+ List<Span> reference = new ArrayList<>();
+ List<Span> prediction = new ArrayList<>();
for (int i = 0; i < refs.length; i++) {
if (refs[i].equals(tag)) {
reference.add(new Span(i, i + 1));
@@ -645,7 +645,7 @@ public class POSTaggerFineGrainedReportListener implements
}
public SortedSet<String> getTokensOrderedByFrequency() {
- SortedSet<String> toks = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> toks = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -668,7 +668,7 @@ public class POSTaggerFineGrainedReportListener implements
}
public SortedSet<String> getTokensOrderedByNumberOfErrors() {
- SortedSet<String> toks = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> toks = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -709,7 +709,7 @@ public class POSTaggerFineGrainedReportListener implements
}
public SortedSet<String> getTagsOrderedByErrors() {
- SortedSet<String> tags = new TreeSet<String>(new Comparator<String>() {
+ SortedSet<String> tags = new TreeSet<>(new Comparator<String>() {
public int compare(String o1, String o2) {
if (o1.equals(o2)) {
return 0;
@@ -764,8 +764,7 @@ public class POSTaggerFineGrainedReportListener implements
column++;
}
// set accuracy
- matrix[line][column] = data.get(ref) != null ? data.get(ref)
- .getAccuracy() : 0;
+ matrix[line][column] = data.get(ref) != null ? data.get(ref).getAccuracy() : 0;
line++;
}
@@ -774,9 +773,9 @@ public class POSTaggerFineGrainedReportListener implements
private SortedSet<String> getConfusionMatrixTagset(
Map<String, ConfusionMatrixLine> data) {
- SortedSet<String> tags = new TreeSet<String>(new CategoryComparator(data));
+ SortedSet<String> tags = new TreeSet<>(new CategoryComparator(data));
tags.addAll(data.keySet());
- List<String> col = new LinkedList<String>();
+ List<String> col = new LinkedList<>();
for (String t : tags) {
col.addAll(data.get(t).line.keySet());
}
@@ -829,7 +828,7 @@ public class POSTaggerFineGrainedReportListener implements
*/
private static class ConfusionMatrixLine {
- private Map<String, Counter> line = new HashMap<String, Counter>();
+ private Map<String, Counter> line = new HashMap<>();
private String ref;
private int total = 0;
private int correct = 0;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTool.java
index d2ceab8..2c4c661 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTool.java
@@ -51,7 +51,7 @@ public final class POSTaggerTool extends BasicCmdLineTool {
POSTaggerME tagger = new POSTaggerME(model);
- ObjectStream<String> lineStream = null;
+ ObjectStream<String> lineStream;
PerformanceMonitor perfMon = null;
try {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTrainerTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTrainerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTrainerTool.java
index 42e4aa1..c04b5ef 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTrainerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerTrainerTool.java
@@ -85,7 +85,7 @@ public final class POSTaggerTrainerTool
System.err.println("done");
}
- POSTaggerFactory postaggerFactory = null;
+ POSTaggerFactory postaggerFactory;
try {
postaggerFactory = POSTaggerFactory.create(params.getFactory(), ngramDict, null);
} catch (InvalidFormatException e) {
@@ -149,17 +149,19 @@ public final class POSTaggerTrainerTool
if (modelString == null)
modelString = "maxent";
- if (modelString.equals("maxent")) {
- model = ModelType.MAXENT;
- }
- else if (modelString.equals("perceptron")) {
- model = ModelType.PERCEPTRON;
- }
- else if (modelString.equals("perceptron_sequence")) {
- model = ModelType.PERCEPTRON_SEQUENCE;
- }
- else {
- model = null;
+ switch (modelString) {
+ case "maxent":
+ model = ModelType.MAXENT;
+ break;
+ case "perceptron":
+ model = ModelType.PERCEPTRON;
+ break;
+ case "perceptron_sequence":
+ model = ModelType.PERCEPTRON_SEQUENCE;
+ break;
+ default:
+ model = null;
+ break;
}
return model;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/CommandLineTokenizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/CommandLineTokenizer.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/CommandLineTokenizer.java
index 2c537a9..4c81d93 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/CommandLineTokenizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/CommandLineTokenizer.java
@@ -36,9 +36,9 @@ final class CommandLineTokenizer {
}
void process() {
- ObjectStream<String> untokenizedLineStream = null;
+ ObjectStream<String> untokenizedLineStream;
- ObjectStream<String> tokenizedLineStream = null;
+ ObjectStream<String> tokenizedLineStream;
PerformanceMonitor perfMon = null;
try {
untokenizedLineStream =
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
index aa1fef8..58b7a6e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
@@ -28,12 +28,10 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
import java.util.StringTokenizer;
-
import opennlp.tools.dictionary.serializer.Attributes;
import opennlp.tools.dictionary.serializer.DictionarySerializer;
import opennlp.tools.dictionary.serializer.Entry;
import opennlp.tools.dictionary.serializer.EntryInserter;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.StringList;
import opennlp.tools.util.StringUtil;
@@ -91,7 +89,7 @@ public class Dictionary implements Iterable<StringList> {
}
}
- private Set<StringListWrapper> entrySet = new HashSet<StringListWrapper>();
+ private Set<StringListWrapper> entrySet = new HashSet<>();
private final boolean isCaseSensitive;
private int minTokenCount = 99999;
private int maxTokenCount = 0;
@@ -113,9 +111,8 @@ public class Dictionary implements Iterable<StringList> {
*
* @param in
* @throws IOException
- * @throws InvalidFormatException
*/
- public Dictionary(InputStream in) throws IOException, InvalidFormatException {
+ public Dictionary(InputStream in) throws IOException {
isCaseSensitive = DictionarySerializer.create(in, new EntryInserter() {
public void insert(Entry entry) {
put(entry.getTokens());
@@ -136,9 +133,8 @@ public class Dictionary implements Iterable<StringList> {
* @param caseSensitive
* has no effect
* @throws IOException
- * @throws InvalidFormatException
*/
- public Dictionary(InputStream in, boolean caseSensitive) throws IOException, InvalidFormatException {
+ public Dictionary(InputStream in, boolean caseSensitive) throws IOException {
this(in);
}
@@ -173,7 +169,6 @@ public class Dictionary implements Iterable<StringList> {
* Checks if this dictionary has the given entry.
*
* @param tokens
- *
* @return true if it contains the entry otherwise false
*/
public boolean contains(StringList tokens) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Index.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/dictionary/Index.java b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Index.java
index df4e9c5..7d1245e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/dictionary/Index.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Index.java
@@ -30,7 +30,7 @@ import opennlp.tools.util.StringList;
*/
public class Index {
- private Set<String> tokens = new HashSet<String>();
+ private Set<String> tokens = new HashSet<>();
/**
* Initializes the current instance with the given
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/Attributes.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/Attributes.java b/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/Attributes.java
index 1b2b488..cd737c8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/Attributes.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/Attributes.java
@@ -30,7 +30,7 @@ import java.util.Map;
*/
public class Attributes {
- private Map<String, String> mNameValueMap = new HashMap<String, String>();
+ private Map<String, String> mNameValueMap = new HashMap<>();
/**
* Retrieves the value for the given key or null if attribute it not set.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/DictionarySerializer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/DictionarySerializer.java b/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/DictionarySerializer.java
index feb96a3..a0b3dcb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/DictionarySerializer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/dictionary/serializer/DictionarySerializer.java
@@ -59,7 +59,7 @@ public class DictionarySerializer {
private boolean mIsInsideTokenElement;
private boolean mIsCaseSensitiveDictionary;
- private List<String> mTokenList = new LinkedList<String>();
+ private List<String> mTokenList = new LinkedList<>();
private StringBuilder token = new StringBuilder();
@@ -209,7 +209,7 @@ public class DictionarySerializer {
* @throws InvalidFormatException
*/
public static boolean create(InputStream in, EntryInserter inserter)
- throws IOException, InvalidFormatException {
+ throws IOException {
DictionaryContenthandler profileContentHandler =
new DictionaryContenthandler(inserter);
@@ -299,8 +299,7 @@ public class DictionarySerializer {
hd.endDocument();
}
catch (SAXException e) {
- //TODO update after Java6 upgrade
- throw (IOException) new IOException("Error during serialization: " + e.getMessage()).initCause(e);
+ throw new IOException("Error during serialization: " + e.getMessage(), e);
}
}
@@ -320,14 +319,11 @@ public class DictionarySerializer {
StringList tokens = entry.getTokens();
- for (Iterator<String> it = tokens.iterator(); it.hasNext(); ) {
+ for (String token1 : tokens) {
hd.startElement("", "", TOKEN_ELEMENT, new AttributesImpl());
- String token = it.next();
-
- hd.characters(token.toCharArray(),
- 0, token.length());
+ hd.characters(token1.toCharArray(), 0, token1.length());
hd.endElement("", "", TOKEN_ELEMENT);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/BagOfWordsFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/BagOfWordsFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/BagOfWordsFeatureGenerator.java
index 848f4e1..ac39afc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/BagOfWordsFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/BagOfWordsFeatureGenerator.java
@@ -41,7 +41,7 @@ public class BagOfWordsFeatureGenerator implements FeatureGenerator {
@Override
public Collection<String> extractFeatures(String[] text, Map<String, Object> extraInformation) {
- Collection<String> bagOfWords = new ArrayList<String>(text.length);
+ Collection<String> bagOfWords = new ArrayList<>(text.length);
for (String word : text) {
if (useOnlyAllLetterTokens) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatCrossValidator.java
index dbce077..106b82d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatCrossValidator.java
@@ -65,7 +65,7 @@ public class DoccatCrossValidator {
public void evaluate(ObjectStream<DocumentSample> samples, int nFolds)
throws IOException {
- CrossValidationPartitioner<DocumentSample> partitioner = new CrossValidationPartitioner<DocumentSample>(
+ CrossValidationPartitioner<DocumentSample> partitioner = new CrossValidationPartitioner<>(
samples, nFolds);
while (partitioner.hasNext()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatFactory.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatFactory.java
index 9b30d95..b95671a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatFactory.java
@@ -140,8 +140,7 @@ public class DoccatFactory extends BaseToolFactory {
}
if (featureGenerators == null) { // could not load using artifact provider
// load bag of words as default
- FeatureGenerator[] bow = {new BagOfWordsFeatureGenerator()};
- this.featureGenerators = bow;
+ this.featureGenerators = new FeatureGenerator[]{new BagOfWordsFeatureGenerator()};
}
}
return featureGenerators;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
index 2bbaace..e71b625 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
@@ -45,15 +45,15 @@ public class DoccatModel extends BaseModel {
checkArtifactMap();
}
- public DoccatModel(InputStream in) throws IOException, InvalidFormatException {
+ public DoccatModel(InputStream in) throws IOException {
super(COMPONENT_NAME, in);
}
- public DoccatModel(File modelFile) throws IOException, InvalidFormatException {
+ public DoccatModel(File modelFile) throws IOException {
super(COMPONENT_NAME, modelFile);
}
- public DoccatModel(URL modelURL) throws IOException, InvalidFormatException {
+ public DoccatModel(URL modelURL) throws IOException {
super(COMPONENT_NAME, modelURL);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerContextGenerator.java
index b62d8eb..737b9d1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerContextGenerator.java
@@ -33,7 +33,7 @@ class DocumentCategorizerContextGenerator {
public String[] getContext(String text[], Map<String, Object> extraInformation) {
- Collection<String> context = new LinkedList<String>();
+ Collection<String> context = new LinkedList<>();
for (FeatureGenerator mFeatureGenerator : mFeatureGenerators) {
Collection<String> extractedFeatures =
[04/50] [abbrv] opennlp git commit: Remove deprecated API from the
POS Tagger
Posted by jo...@apache.org.
Remove deprecated API from the POS Tagger
See issue OPENNLP-883
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/639b9f0a
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/639b9f0a
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/639b9f0a
Branch: refs/heads/889
Commit: 639b9f0ae2200d7a7365eeb43227ca067330bc83
Parents: c657cde
Author: J�rn Kottmann <jo...@apache.org>
Authored: Tue Nov 22 01:39:50 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Tue Nov 22 01:39:50 2016 +0100
----------------------------------------------------------------------
.../opennlp/tools/postag/POSDictionary.java | 76 --------------
.../java/opennlp/tools/postag/POSModel.java | 55 ----------
.../java/opennlp/tools/postag/POSTagger.java | 29 -----
.../tools/postag/POSTaggerCrossValidator.java | 35 -------
.../java/opennlp/tools/postag/POSTaggerME.java | 105 +------------------
.../opennlp/tools/postag/POSTaggerMETest.java | 9 +-
.../java/opennlp/uima/postag/POSTagger.java | 8 +-
.../opennlp/uima/postag/POSTaggerTrainer.java | 22 ++--
8 files changed, 23 insertions(+), 316 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java
index 7904d83..ef237a8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSDictionary.java
@@ -65,82 +65,6 @@ public class POSDictionary implements Iterable<String>, MutableTagDictionary {
}
/**
- * Creates a tag dictionary with contents of specified file.
- *
- * @param file The file name for the tag dictionary.
- *
- * @throws IOException when the specified file can not be read.
- *
- * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
- */
- @Deprecated
- public POSDictionary(String file) throws IOException {
- this(file, null, true);
- }
-
- /**
- * Creates a tag dictionary with contents of specified file and using specified
- * case to determine how to access entries in the tag dictionary.
- *
- * @param file The file name for the tag dictionary.
- * @param caseSensitive Specifies whether the tag dictionary is case sensitive or not.
- *
- * @throws IOException when the specified file can not be read.
- *
- * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
- */
- @Deprecated
- public POSDictionary(String file, boolean caseSensitive) throws IOException {
- this(file, null, caseSensitive);
- }
-
-
- /**
- * Creates a tag dictionary with contents of specified file and using specified case to determine how to access entries in the tag dictionary.
- *
- * @param file The file name for the tag dictionary.
- * @param encoding The encoding of the tag dictionary file.
- * @param caseSensitive Specifies whether the tag dictionary is case sensitive or not.
- *
- * @throws IOException when the specified file can not be read.
- *
- * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
- */
- @Deprecated
- public POSDictionary(String file, String encoding, boolean caseSensitive) throws IOException {
- this(new BufferedReader(encoding == null ? new FileReader(file) : new InputStreamReader(new FileInputStream(file),encoding)), caseSensitive);
- }
-
- /**
- * Create tag dictionary object with contents of specified file and using specified case to determine how to access entries in the tag dictionary.
- *
- * @param reader A reader for the tag dictionary.
- * @param caseSensitive Specifies whether the tag dictionary is case sensitive or not.
- *
- * @throws IOException when the specified file can not be read.
- *
- * @deprecated Use {@link POSDictionary#create(InputStream)} instead, old format might removed.
- */
- @Deprecated
- public POSDictionary(BufferedReader reader, boolean caseSensitive) throws IOException {
- dictionary = new HashMap<String, String[]>();
- this.caseSensitive = caseSensitive;
- for (String line = reader.readLine(); line != null; line = reader.readLine()) {
- String[] parts = line.split(" ");
- String[] tags = new String[parts.length - 1];
- for (int ti = 0, tl = parts.length - 1; ti < tl; ti++) {
- tags[ti] = parts[ti + 1];
- }
- if (caseSensitive) {
- dictionary.put(parts[0], tags);
- }
- else {
- dictionary.put(StringUtil.toLowerCase(parts[0]), tags);
- }
- }
- }
-
- /**
* Returns a list of valid tags for the specified word.
*
* @param word The word.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
index 446d1e6..34b9f79 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
@@ -45,29 +45,6 @@ public final class POSModel extends BaseModel {
public static final String POS_MODEL_ENTRY_NAME = "pos.model";
- /**
- * @deprecated Use
- * {@link #POSModel(String, MaxentModel, Map, POSTaggerFactory)}
- * instead.
- */
- public POSModel(String languageCode, MaxentModel posModel,
- POSDictionary tagDictionary, Dictionary ngramDict, Map<String, String> manifestInfoEntries) {
-
- this(languageCode, posModel, manifestInfoEntries, new POSTaggerFactory(
- ngramDict, tagDictionary));
- }
-
- /**
- * @deprecated Use
- * {@link #POSModel(String, MaxentModel, Map, POSTaggerFactory)}
- * instead.
- */
- public POSModel(String languageCode, MaxentModel posModel,
- POSDictionary tagDictionary, Dictionary ngramDict) {
- this(languageCode, posModel, POSTaggerME.DEFAULT_BEAM_SIZE, null, new POSTaggerFactory(ngramDict,
- tagDictionary));
- }
-
public POSModel(String languageCode, SequenceClassificationModel<String> posModel,
Map<String, String> manifestInfoEntries, POSTaggerFactory posFactory) {
@@ -139,7 +116,6 @@ public final class POSModel extends BaseModel {
* @deprecated use getPosSequenceModel instead. This method will be removed soon.
*/
@Deprecated
-
public MaxentModel getPosModel() {
if (artifactMap.get(POS_MODEL_ENTRY_NAME) instanceof MaxentModel) {
return (MaxentModel) artifactMap.get(POS_MODEL_ENTRY_NAME);
@@ -171,37 +147,6 @@ public final class POSModel extends BaseModel {
}
}
- /**
- * Retrieves the tag dictionary.
- *
- * @return tag dictionary or null if not used
- *
- * @deprecated Use {@link POSModel#getFactory()} to get a
- * {@link POSTaggerFactory} and
- * {@link POSTaggerFactory#getTagDictionary()} to get a
- * {@link TagDictionary}.
- *
- * @throws IllegalStateException
- * if the TagDictionary is not an instance of POSDictionary
- */
- public POSDictionary getTagDictionary() {
- if (getFactory() != null) {
- TagDictionary dict = getFactory().getTagDictionary();
- if (dict != null) {
- if (dict instanceof POSDictionary) {
- return (POSDictionary) dict;
- }
- String clazz = dict.getClass().getCanonicalName();
- throw new IllegalStateException("Can not get a dictionary of type "
- + clazz
- + " using the deprecated method POSModel.getTagDictionary() "
- + "because it can only return dictionaries of type POSDictionary. "
- + "Use POSModel.getFactory().getTagDictionary() instead.");
- }
- }
- return null;
- }
-
public POSTaggerFactory getFactory() {
return (POSTaggerFactory) this.toolFactory;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
index 3cfc522..f081916 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
@@ -28,18 +28,6 @@ public interface POSTagger {
/**
* Assigns the sentence of tokens pos tags.
- *
- * @param sentence
- * The sentence of tokens to be tagged.
- * @return a list of pos tags for each token provided in sentence.
- *
- * @deprecated call <code> tag(String[]) </code> instead
- */
- @Deprecated
- public List<String> tag(List<String> sentence);
-
- /**
- * Assigns the sentence of tokens pos tags.
* @param sentence The sentece of tokens to be tagged.
* @return an array of pos tags for each token provided in sentence.
*/
@@ -47,23 +35,6 @@ public interface POSTagger {
public String[] tag(String[] sentence, Object[] additionaContext);
- /**
- * Assigns the sentence of space-delimied tokens pos tags.
- * @param sentence The sentece of space-delimited tokens to be tagged.
- * @return a string of space-delimited pos tags for each token provided in sentence.
- *
- * @deprecated call <code> tag(String[]) instead </code> use WhiteSpaceTokenizer.INSTANCE.tokenize
- * to obtain the String array.
- */
- @Deprecated
- public String tag(String sentence);
-
- /**
- * @deprecated call <code> topKSequences(String[]) </code> instead
- */
- @Deprecated
- public Sequence[] topKSequences(List<String> sentence);
-
public Sequence[] topKSequences(String[] sentence);
public Sequence[] topKSequences(String[] sentence, Object[] additionaContext);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
index c767268..27854dc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
@@ -79,41 +79,6 @@ public class POSTaggerCrossValidator {
}
/**
- * @deprecated use
- * {@link #POSTaggerCrossValidator(String, TrainingParameters, POSTaggerFactory, POSTaggerEvaluationMonitor...)}
- * instead and pass in a {@link POSTaggerFactory}.
- */
- public POSTaggerCrossValidator(String languageCode,
- TrainingParameters trainParam, POSDictionary tagDictionary,
- POSTaggerEvaluationMonitor... listeners) {
- this(languageCode, trainParam, create(null, tagDictionary), listeners);
- }
-
- /**
- * @deprecated use
- * {@link #POSTaggerCrossValidator(String, TrainingParameters, POSTaggerFactory, POSTaggerEvaluationMonitor...)}
- * instead and pass in the name of {@link POSTaggerFactory}
- * sub-class.
- */
- public POSTaggerCrossValidator(String languageCode,
- TrainingParameters trainParam, POSDictionary tagDictionary,
- Integer ngramCutoff, POSTaggerEvaluationMonitor... listeners) {
- this(languageCode, trainParam, create(null, tagDictionary), listeners);
- this.ngramCutoff = ngramCutoff;
- }
-
- /**
- * @deprecated use
- * {@link #POSTaggerCrossValidator(String, TrainingParameters, POSTaggerFactory, POSTaggerEvaluationMonitor...)}
- * instead and pass in a {@link POSTaggerFactory}.
- */
- public POSTaggerCrossValidator(String languageCode,
- TrainingParameters trainParam, POSDictionary tagDictionary,
- Dictionary ngramDictionary, POSTaggerEvaluationMonitor... listeners) {
- this(languageCode, trainParam, create(ngramDictionary, tagDictionary), listeners);
- }
-
- /**
* Starts the evaluation.
*
* @param samples
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
index e2e5188..e4c1c1b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
@@ -90,39 +90,7 @@ public class POSTaggerME implements POSTagger {
private SequenceValidator<String> sequenceValidator;
/**
- * Initializes the current instance with the provided
- * model and provided beam size.
- *
- * @param model
- * @param beamSize
- *
- * @deprecated the beam size should be specified in the params during training
- */
- @Deprecated
- public POSTaggerME(POSModel model, int beamSize, int cacheSize) {
- POSTaggerFactory factory = model.getFactory();
-
- modelPackage = model;
-
- // TODO: Why is this the beam size?! not cache size?
- contextGen = factory.getPOSContextGenerator(beamSize);
- tagDictionary = factory.getTagDictionary();
- size = beamSize;
-
- sequenceValidator = factory.getSequenceValidator();
-
- if (model.getPosSequenceModel() != null) {
- this.model = model.getPosSequenceModel();
- }
- else {
- this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
- model.getPosModel(), cacheSize);
- }
- }
-
- /**
- * Initializes the current instance with the provided model
- * and the default beam size of 3.
+ * Initializes the current instance with the provided model.
*
* @param model
*/
@@ -156,21 +124,6 @@ public class POSTaggerME implements POSTagger {
}
/**
- * Returns the number of different tags predicted by this model.
- *
- * @return the number of different tags predicted by this model.
- * @deprecated use getAllPosTags instead!
- */
- @Deprecated
- public int getNumTags() {
-
- // TODO: Lets discuss on the dev list how to do this properly!
- // Nobody needs the number of tags, if the tags are not available.
-
- return model.getOutcomes().length;
- }
-
- /**
* Retrieves an array of all possible part-of-speech tags from the
* tagger.
*
@@ -180,12 +133,6 @@ public class POSTaggerME implements POSTagger {
return model.getOutcomes();
}
- @Deprecated
- public List<String> tag(List<String> sentence) {
- bestSequence = model.bestSequence(sentence.toArray(new String[sentence.size()]), null, contextGen, sequenceValidator);
- return bestSequence.getOutcomes();
- }
-
public String[] tag(String[] sentence) {
return this.tag(sentence, null);
}
@@ -215,12 +162,6 @@ public class POSTaggerME implements POSTagger {
return tags;
}
- @Deprecated
- public Sequence[] topKSequences(List<String> sentence) {
- return model.bestSequences(size, sentence.toArray(new String[sentence.size()]), null,
- contextGen, sequenceValidator);
- }
-
public Sequence[] topKSequences(String[] sentence) {
return this.topKSequences(sentence, null);
}
@@ -247,19 +188,6 @@ public class POSTaggerME implements POSTagger {
return bestSequence.getProbs();
}
- @Deprecated
- public String tag(String sentence) {
- List<String> toks = new ArrayList<String>();
- StringTokenizer st = new StringTokenizer(sentence);
- while (st.hasMoreTokens())
- toks.add(st.nextToken());
- List<String> tags = tag(toks);
- StringBuilder sb = new StringBuilder();
- for (int i = 0; i < tags.size(); i++)
- sb.append(toks.get(i) + "/" + tags.get(i) + " ");
- return sb.toString().trim();
- }
-
public String[] getOrderedTags(List<String> words, List<String> tags, int index) {
return getOrderedTags(words,tags,index,null);
}
@@ -349,37 +277,6 @@ public class POSTaggerME implements POSTagger {
}
}
- /**
- * @deprecated use
- * {@link #train(String, ObjectStream, TrainingParameters, POSTaggerFactory)}
- * instead and pass in a {@link POSTaggerFactory}.
- */
- public static POSModel train(String languageCode, ObjectStream<POSSample> samples, TrainingParameters trainParams,
- POSDictionary tagDictionary, Dictionary ngramDictionary) throws IOException {
-
- return train(languageCode, samples, trainParams, new POSTaggerFactory(
- ngramDictionary, tagDictionary));
- }
-
- /**
- * @deprecated use
- * {@link #train(String, ObjectStream, TrainingParameters, POSTaggerFactory)}
- * instead and pass in a {@link POSTaggerFactory} and a
- * {@link TrainingParameters}.
- */
- @Deprecated
- public static POSModel train(String languageCode, ObjectStream<POSSample> samples, ModelType modelType, POSDictionary tagDictionary,
- Dictionary ngramDictionary, int cutoff, int iterations) throws IOException {
-
- TrainingParameters params = new TrainingParameters();
-
- params.put(TrainingParameters.ALGORITHM_PARAM, modelType.toString());
- params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(cutoff));
-
- return train(languageCode, samples, params, tagDictionary, ngramDictionary);
- }
-
public static Dictionary buildNGramDictionary(ObjectStream<POSSample> samples, int cutoff)
throws IOException {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
index 1d99687..996b233 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import opennlp.tools.util.TrainingParameters;
import org.junit.Test;
import opennlp.tools.formats.ResourceAsStreamFactory;
@@ -50,8 +51,12 @@ public class POSTaggerMETest {
* @throws IOException
*/
static POSModel trainPOSModel(ModelType type) throws IOException {
- // TODO: also use tag dictionary for training
- return POSTaggerME.train("en", createSampleStream(), type, null, null, 5, 100);
+ TrainingParameters params = new TrainingParameters();
+ params.put(TrainingParameters.ALGORITHM_PARAM, type.toString());
+ params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
+ params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(5));
+
+ return POSTaggerME.train("en", createSampleStream(), params, new POSTaggerFactory());
}
@Test
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
index 5e77e9d..2fdc47c 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTagger.java
@@ -17,6 +17,7 @@
package opennlp.uima.postag;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -127,7 +128,7 @@ public final class POSTagger extends CasAnnotator_ImplBase {
if (beamSize == null)
beamSize = POSTaggerME.DEFAULT_BEAM_SIZE;
- this.posTagger = new POSTaggerME(model, beamSize, 0);
+ this.posTagger = new POSTaggerME(model);
}
/**
@@ -174,7 +175,8 @@ public final class POSTagger extends CasAnnotator_ImplBase {
sentenceTokenList.add(tokenAnnotation.getCoveredText());
}
- final List<String> posTags = this.posTagger.tag(sentenceTokenList);
+ final List<String> posTags = Arrays.asList(this.posTagger.tag(
+ sentenceTokenList.toArray(new String[sentenceTokenList.size()])));
double posProbabilities[] = null;
@@ -231,4 +233,4 @@ public final class POSTagger extends CasAnnotator_ImplBase {
public void destroy() {
this.posTagger = null;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/639b9f0a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
index 9f377be..e9bb048 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
@@ -27,11 +27,9 @@ import java.util.Iterator;
import java.util.List;
import opennlp.tools.ml.maxent.GIS;
-import opennlp.tools.postag.POSDictionary;
-import opennlp.tools.postag.POSModel;
-import opennlp.tools.postag.POSSample;
-import opennlp.tools.postag.POSTaggerME;
+import opennlp.tools.postag.*;
import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelType;
import opennlp.uima.util.AnnotatorUtil;
import opennlp.uima.util.CasConsumerUtil;
@@ -116,12 +114,8 @@ public class POSTaggerTrainer extends CasConsumer_ImplBase {
TAG_DICTIONARY_NAME);
if (tagDictionaryName != null) {
- try {
- InputStream dictIn = AnnotatorUtil.getResourceAsStream(mContext, tagDictionaryName);
-
- // TODO: ask Tom if case sensitivity must be configureable
- tagDictionary = new POSDictionary(new BufferedReader(new InputStreamReader(dictIn)), false);
-
+ try (InputStream dictIn = AnnotatorUtil.getResourceAsStream(mContext, tagDictionaryName)) {
+ tagDictionary = POSDictionary.create(dictIn);
} catch (final IOException e) {
// if this fails just print error message and continue
final String message = "IOException during tag dictionary reading, "
@@ -207,9 +201,13 @@ public class POSTaggerTrainer extends CasConsumer_ImplBase {
GIS.PRINT_MESSAGES = false;
+ TrainingParameters params = new TrainingParameters();
+ params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
+ params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(5));
+
POSModel posTaggerModel = POSTaggerME.train(language,
ObjectStreamUtils.createObjectStream(mPOSSamples),
- ModelType.MAXENT, tagDictionary, null, 100, 5);
+ params, new POSTaggerFactory(null, tagDictionary));
// dereference to allow garbage collection
mPOSSamples = null;
@@ -234,4 +232,4 @@ public class POSTaggerTrainer extends CasConsumer_ImplBase {
// dereference to allow garbage collection
mPOSSamples = null;
}
-}
\ No newline at end of file
+}
[39/50] [abbrv] opennlp git commit: Replace manual array copy
Posted by jo...@apache.org.
Replace manual array copy
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/dd2aaccc
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/dd2aaccc
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/dd2aaccc
Branch: refs/heads/889
Commit: dd2aaccc0737eace57a949f2d7d7291bddb81ee9
Parents: 06c27b3
Author: Joern Kottmann <jo...@apache.org>
Authored: Wed Dec 21 17:12:52 2016 +0100
Committer: Joern Kottmann <jo...@apache.org>
Committed: Wed Dec 21 17:12:52 2016 +0100
----------------------------------------------------------------------
.../src/main/java/opennlp/tools/ml/maxent/GISTrainer.java | 4 +---
.../main/java/opennlp/tools/ml/model/AbstractModelReader.java | 4 +---
.../main/java/opennlp/tools/parser/AbstractBottomUpParser.java | 4 +---
.../java/opennlp/tools/parser/chunking/ParserEventStream.java | 4 +---
4 files changed, 4 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd2aaccc/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
index 9919bb0..05a5424 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
@@ -361,9 +361,7 @@ class GISTrainer {
}
else {
outcomePattern = new int[numActiveOutcomes];
- for (int aoi=0;aoi<numActiveOutcomes;aoi++) {
- outcomePattern[aoi] = activeOutcomes[aoi];
- }
+ System.arraycopy(activeOutcomes, 0, outcomePattern, 0, numActiveOutcomes);
}
}
params[pi] = new MutableContext(outcomePattern,new double[numActiveOutcomes]);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd2aaccc/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java
index 6a4b642..6c26214 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModelReader.java
@@ -135,9 +135,7 @@ public abstract class AbstractModelReader {
for (int i=0; i<outcomePatterns.length; i++) {
//construct outcome pattern
int[] outcomePattern = new int[outcomePatterns[i].length-1];
- for (int k=1; k<outcomePatterns[i].length; k++) {
- outcomePattern[k-1] = outcomePatterns[i][k];
- }
+ System.arraycopy(outcomePatterns[i], 1, outcomePattern, 0, outcomePatterns[i].length - 1);
//System.err.println("outcomePattern "+i+" of "+outcomePatterns.length+" with "+outcomePatterns[i].length+" outcomes ");
//populate parameters for each context which uses this outcome pattern.
for (int j=0; j<outcomePatterns[i][0]; j++) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd2aaccc/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
index 10c3f0e..ff4bea8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
@@ -576,9 +576,7 @@ public abstract class AbstractBottomUpParser implements Parser {
if (ci+2 < chunks.length) window[wi++] = chunks[ci+2].getHead().getCoveredText();
if (wi < 5) {
String[] subWindow = new String[wi];
- for (int swi=0;swi<wi;swi++) {
- subWindow[swi]=window[swi];
- }
+ System.arraycopy(window, 0, subWindow, 0, wi);
window = subWindow;
}
if (window.length >=3) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd2aaccc/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
index afaed99..8ded9ec 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
@@ -93,9 +93,7 @@ public class ParserEventStream extends AbstractParserEventStream {
if (!type.equals(AbstractBottomUpParser.TOP_NODE)) {
reducedChunks = new Parse[chunks.length-(reduceEnd-reduceStart+1)+1]; //total - num_removed + 1 (for new node)
//insert nodes before reduction
- for (int ri = 0; ri< reduceStart; ri++) {
- reducedChunks[ri]=chunks[ri];
- }
+ System.arraycopy(chunks, 0, reducedChunks, 0, reduceStart);
//insert reduced node
reducedChunks[reduceStart]=parent;
//propagate punctuation sets
[38/50] [abbrv] opennlp git commit: Use String.join instead of manual
join
Posted by jo...@apache.org.
Use String.join instead of manual join
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/06c27b37
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/06c27b37
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/06c27b37
Branch: refs/heads/889
Commit: 06c27b37ccbde70514e035a50ba0a6ee1ff94ea3
Parents: 2b0658c
Author: Joern Kottmann <jo...@apache.org>
Authored: Wed Dec 21 12:06:41 2016 +0100
Committer: Joern Kottmann <jo...@apache.org>
Committed: Wed Dec 21 12:06:41 2016 +0100
----------------------------------------------------------------------
.../tools/eval/SourceForgeModelEval.java | 26 +++-----------------
1 file changed, 4 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06c27b37/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
index 1e7d3ee..639c7fc 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
@@ -17,7 +17,6 @@
package opennlp.tools.eval;
-import junit.framework.Assert;
import opennlp.tools.chunker.Chunker;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
@@ -46,6 +45,7 @@ import opennlp.tools.util.MarkableFileInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
+import org.junit.Assert;
import org.junit.Test;
import java.io.File;
@@ -124,10 +124,7 @@ public class SourceForgeModelEval {
DocumentSample lineBatch ;
while ((lineBatch = lineBatches.read()) != null) {
- // TODO: Replace with Java 8 join
- for (String token : lineBatch.getText()) {
- text.append(token).append(" ");
- }
+ text.append(String.join(" ", lineBatch.getText())).append(" ");
}
}
@@ -162,14 +159,7 @@ public class SourceForgeModelEval {
DocumentSample line;
while ((line = lines.read()) != null) {
-
- // TODO: Replace with Java 8 join
- StringBuffer text = new StringBuffer();
- for (String token : line.getText()) {
- text.append(token).append(' ');
- }
-
- String[] tokens = tokenizer.tokenize(text.toString());
+ String[] tokens = tokenizer.tokenize(String.join(" ", line.getText()));
for (String token : tokens) {
digest.update(token.getBytes("UTF-8"));
}
@@ -345,15 +335,7 @@ public class SourceForgeModelEval {
DocumentSample line;
while ((line = lines.read()) != null) {
-
- StringBuilder textLine = new StringBuilder();
-
- // TODO: Replace with Java 8 join
- for (String token : line.getText()) {
- textLine.append(token).append(' ');
- }
-
- Parse[] parse = ParserTool.parseLine(textLine.toString(), parser, 1);
+ Parse[] parse = ParserTool.parseLine(String.join(" ", line.getText()), parser, 1);
if (parse.length > 0) {
digest.update(parse[0].toString().getBytes("UTF-8"));
}
[49/50] [abbrv] opennlp git commit: OPENNLP-901:Replace references to
deprecated NameFinderME.train(), this closes apache/opennlp#21
Posted by jo...@apache.org.
OPENNLP-901:Replace references to deprecated NameFinderME.train(), this closes apache/opennlp#21
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/dfbf6148
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/dfbf6148
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/dfbf6148
Branch: refs/heads/889
Commit: dfbf61485179f5c020a4f4a5e57db61451ad3c37
Parents: f781fca
Author: smarthi <sm...@apache.org>
Authored: Sun Dec 25 14:25:12 2016 -0500
Committer: smarthi <sm...@apache.org>
Committed: Sun Dec 25 14:25:12 2016 -0500
----------------------------------------------------------------------
.../java/opennlp/morfologik/cmdline/CLI.java | 4 +-
.../TokenNameFinderCrossValidatorTool.java | 4 +-
.../namefind/TokenNameFinderTrainerTool.java | 11 +-
.../opennlp/tools/namefind/NameFinderME.java | 141 +------------------
.../namefind/TokenNameFinderCrossValidator.java | 13 +-
.../tools/namefind/TokenNameFinderFactory.java | 38 +++--
.../tools/namefind/TokenNameFinderModel.java | 64 +--------
.../opennlp/tools/postag/POSTaggerFactory.java | 13 +-
.../tools/namefind/NameFinderMETest.java | 46 +++---
.../TokenNameFinderCrossValidatorTest.java | 13 +-
10 files changed, 68 insertions(+), 279 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
----------------------------------------------------------------------
diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
index f92d178..5205739 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/cmdline/CLI.java
@@ -40,9 +40,9 @@ public final class CLI {
private static Map<String, CmdLineTool> toolLookupMap;
static {
- toolLookupMap = new LinkedHashMap<String, CmdLineTool>();
+ toolLookupMap = new LinkedHashMap<>();
- List<CmdLineTool> tools = new LinkedList<CmdLineTool>();
+ List<CmdLineTool> tools = new LinkedList<>();
tools.add(new MorfologikDictionaryBuilderTool());
tools.add(new XMLDictionaryToTableTool());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
index 93f52ec..aa1e343 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
@@ -73,7 +73,7 @@ public final class TokenNameFinderCrossValidatorTool
sampleStream = new NameSampleTypeFilter(nameTypes, sampleStream);
}
- List<EvaluationMonitor<NameSample>> listeners = new LinkedList<EvaluationMonitor<NameSample>>();
+ List<EvaluationMonitor<NameSample>> listeners = new LinkedList<>();
if (params.getMisclassified()) {
listeners.add(new NameEvaluationErrorListener());
}
@@ -94,7 +94,7 @@ public final class TokenNameFinderCrossValidatorTool
SequenceCodec<String> sequenceCodec = TokenNameFinderFactory.instantiateSequenceCodec(sequenceCodecImplName);
- TokenNameFinderFactory nameFinderFactory = null;
+ TokenNameFinderFactory nameFinderFactory;
try {
nameFinderFactory = TokenNameFinderFactory.create(params.getFactory(),
featureGeneratorBytes, resources, sequenceCodec);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
index 1f8a365..b2ccfc5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
@@ -89,13 +89,13 @@ public final class TokenNameFinderTrainerTool
* @return a map consisting of the file name of the resource and its corresponding Object
*/
public static Map<String, Object> loadResources(File resourcePath, File featureGenDescriptor) {
- Map<String, Object> resources = new HashMap<String, Object>();
+ Map<String, Object> resources = new HashMap<>();
if (resourcePath != null) {
Map<String, ArtifactSerializer> artifactSerializers = TokenNameFinderModel
.createArtifactSerializers();
- List<Element> elements = new ArrayList<Element>();
+ List<Element> elements = new ArrayList<>();
ArtifactSerializer serializer = null;
@@ -134,9 +134,6 @@ public final class TokenNameFinderTrainerTool
try (InputStream resourceIn = CmdLineUtil.openInFile(resourceFile)) {
resources.put(resourceName, serializer.create(resourceIn));
- } catch (InvalidFormatException e) {
- // TODO: Fix exception handling
- e.printStackTrace();
} catch (IOException e) {
// TODO: Fix exception handling
e.printStackTrace();
@@ -160,7 +157,7 @@ public final class TokenNameFinderTrainerTool
return loadResources(resourcePath, featureGeneratorDescriptor);
}
- return new HashMap<String, Object>();
+ return new HashMap<>();
}
public void run(String format, String[] args) {
@@ -200,7 +197,7 @@ public final class TokenNameFinderTrainerTool
SequenceCodec<String> sequenceCodec = TokenNameFinderFactory.instantiateSequenceCodec(sequenceCodecImplName);
- TokenNameFinderFactory nameFinderFactory = null;
+ TokenNameFinderFactory nameFinderFactory;
try {
nameFinderFactory = TokenNameFinderFactory.create(params.getFactory(),
featureGeneratorBytes, resources, sequenceCodec);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
index ff8c143..69f7a4e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+
import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.EventModelSequenceTrainer;
import opennlp.tools.ml.EventTrainer;
@@ -43,14 +44,7 @@ import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
import opennlp.tools.util.featuregen.AdditionalContextFeatureGenerator;
-import opennlp.tools.util.featuregen.BigramNameFeatureGenerator;
-import opennlp.tools.util.featuregen.CachedFeatureGenerator;
import opennlp.tools.util.featuregen.GeneratorFactory;
-import opennlp.tools.util.featuregen.OutcomePriorFeatureGenerator;
-import opennlp.tools.util.featuregen.PreviousMapFeatureGenerator;
-import opennlp.tools.util.featuregen.SentenceFeatureGenerator;
-import opennlp.tools.util.featuregen.TokenClassFeatureGenerator;
-import opennlp.tools.util.featuregen.TokenFeatureGenerator;
import opennlp.tools.util.featuregen.WindowFeatureGenerator;
/**
@@ -91,21 +85,6 @@ public class NameFinderME implements TokenNameFinder {
new WindowFeatureGenerator(additionalContextFeatureGenerator, 8, 8));
}
- @Deprecated
- /*
- * @deprecated the default feature generation is now always included in the models and loaded
- * if not by the factory. Subclasses using this methods should do the same.
- */
- static AdaptiveFeatureGenerator createFeatureGenerator() {
- return new CachedFeatureGenerator(
- new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
- new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
- new OutcomePriorFeatureGenerator(),
- new PreviousMapFeatureGenerator(),
- new BigramNameFeatureGenerator(),
- new SentenceFeatureGenerator(true, false));
- }
-
private static AdaptiveFeatureGenerator createFeatureGenerator(
byte[] generatorDescriptor, final Map<String, Object> resources)
throws IOException {
@@ -289,124 +268,6 @@ public class NameFinderME implements TokenNameFinder {
}
/**
- * Trains a name finder model.
- *
- * @param languageCode the language of the training data
- * @param type null or an override type for all types in the training data
- * @param samples the training data
- * @param trainParams machine learning train parameters
- * @param generator null or the feature generator
- * @param resources the resources for the name finder or null if none
- *
- * @return the newly trained model
- *
- * @throws IOException
- * @deprecated use
- * {@link NameFinderME#train(String, String, ObjectStream, TrainingParameters, TokenNameFinderFactory)}
- * instead.
- */
- @Deprecated
- static TokenNameFinderModel train(String languageCode, String type, ObjectStream<NameSample> samples,
- TrainingParameters trainParams, AdaptiveFeatureGenerator generator, final Map<String, Object> resources)
- throws IOException {
-
- if (languageCode == null) {
- throw new IllegalArgumentException("languageCode must not be null!");
- }
-
- String beamSizeString = trainParams.getSettings().get(BeamSearch.BEAM_SIZE_PARAMETER);
-
- int beamSize = NameFinderME.DEFAULT_BEAM_SIZE;
- if (beamSizeString != null) {
- beamSize = Integer.parseInt(beamSizeString);
- }
-
- Map<String, String> manifestInfoEntries = new HashMap<>();
-
- AdaptiveFeatureGenerator featureGenerator;
-
- if (generator != null) {
- featureGenerator = generator;
- } else {
- featureGenerator = createFeatureGenerator();
- }
-
- MaxentModel nameFinderModel = null;
-
- SequenceClassificationModel<String> seqModel = null;
-
- TrainerType trainerType = TrainerFactory.getTrainerType(trainParams.getSettings());
-
- if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) {
- ObjectStream<Event> eventStream = new NameFinderEventStream(samples, type,
- new DefaultNameContextGenerator(featureGenerator), new BioCodec());
-
- EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams.getSettings(), manifestInfoEntries);
- nameFinderModel = trainer.train(eventStream);
- } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) {
- NameSampleSequenceStream ss = new NameSampleSequenceStream(samples, featureGenerator);
-
- EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(
- trainParams.getSettings(), manifestInfoEntries);
- nameFinderModel = trainer.train(ss);
- } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) {
- SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer(
- trainParams.getSettings(), manifestInfoEntries);
-
- NameSampleSequenceStream ss = new NameSampleSequenceStream(samples, featureGenerator, false);
- seqModel = trainer.train(ss);
- } else {
- throw new IllegalStateException("Unexpected trainer type!");
- }
-
- // TODO: Pass the sequence codec down to the model! We will just store the class
- // name in the model, and then always use the extension loader to create it!
- // The cmd line interface, will replace shortcuts with actual class names.
- // depending on which one is not null!
- if (seqModel != null) {
- return new TokenNameFinderModel(languageCode, seqModel, null,
- resources, manifestInfoEntries, new BioCodec(), new TokenNameFinderFactory());
- } else {
- return new TokenNameFinderModel(languageCode, nameFinderModel, beamSize, null,
- resources, manifestInfoEntries, new BioCodec(), new TokenNameFinderFactory());
- }
- }
-
- /**
- * Trains a name finder model.
- *
- * @param languageCode the language of the training data
- * @param type null or an override type for all types in the training data
- * @param samples the training data
- * @param trainParams machine learning train parameters
- * @param featureGeneratorBytes descriptor to configure the feature generation
- * or null
- * @param resources the resources for the name finder or null if none
- *
- * @return the newly trained model
- *
- * @throws IOException
- * @deprecated use
- * {@link NameFinderME#train(String, String, ObjectStream, TrainingParameters, TokenNameFinderFactory)}
- * instead.
- */
- @Deprecated
- static TokenNameFinderModel train(String languageCode, String type,
- ObjectStream<NameSample> samples, TrainingParameters trainParams,
- byte[] featureGeneratorBytes, final Map<String, Object> resources)
- throws IOException {
-
- TokenNameFinderModel model = train(languageCode, type, samples, trainParams,
- createFeatureGenerator(featureGeneratorBytes, resources), resources);
-
- if (featureGeneratorBytes != null) {
- model = model.updateFeatureGenerator(featureGeneratorBytes);
- }
-
- return model;
- }
-
- /**
* Gets the name type from the outcome
*
* @param outcome the outcome
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
index 3d2547b..b4ff4e1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
@@ -170,9 +170,7 @@ public class TokenNameFinderCrossValidator {
this.type = type;
this.featureGeneratorBytes = featureGeneratorBytes;
this.resources = resources;
-
this.params = trainParams;
-
this.listeners = listeners;
}
@@ -212,17 +210,16 @@ public class TokenNameFinderCrossValidator {
while (partitioner.hasNext()) {
- CrossValidationPartitioner.TrainingSampleStream<DocumentSample> trainingSampleStream = partitioner
- .next();
+ CrossValidationPartitioner.TrainingSampleStream<DocumentSample> trainingSampleStream = partitioner.next();
TokenNameFinderModel model;
if (factory != null) {
- model = opennlp.tools.namefind.NameFinderME.train(languageCode, type, new DocumentToNameSampleStream(trainingSampleStream), params, factory);
+ model = NameFinderME.train(languageCode, type, new DocumentToNameSampleStream(trainingSampleStream),
+ params, factory);
}
else {
- model = opennlp.tools.namefind.NameFinderME.train(languageCode, type,
- new DocumentToNameSampleStream(trainingSampleStream), params, featureGeneratorBytes, resources);
-
+ model = NameFinderME.train(languageCode, type, new DocumentToNameSampleStream(trainingSampleStream),
+ params, TokenNameFinderFactory.create(null, featureGeneratorBytes, resources, new BioCodec()));
}
// do testing
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
index 90381ff..55f1ab6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
@@ -30,12 +30,19 @@ import opennlp.tools.util.SequenceCodec;
import opennlp.tools.util.ext.ExtensionLoader;
import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
import opennlp.tools.util.featuregen.AggregatedFeatureGenerator;
-import opennlp.tools.util.featuregen.FeatureGeneratorResourceProvider;
+import opennlp.tools.util.featuregen.BigramNameFeatureGenerator;
+import opennlp.tools.util.featuregen.CachedFeatureGenerator;
import opennlp.tools.util.featuregen.GeneratorFactory;
+import opennlp.tools.util.featuregen.OutcomePriorFeatureGenerator;
+import opennlp.tools.util.featuregen.PreviousMapFeatureGenerator;
+import opennlp.tools.util.featuregen.SentenceFeatureGenerator;
+import opennlp.tools.util.featuregen.TokenClassFeatureGenerator;
+import opennlp.tools.util.featuregen.TokenFeatureGenerator;
+import opennlp.tools.util.featuregen.WindowFeatureGenerator;
// Idea of this factory is that most resources/impls used by the name finder
// can be modified through this class!
-// That only works if thats the central class used for training/runtime
+// That only works if that's the central class used for training/runtime
public class TokenNameFinderFactory extends BaseToolFactory {
@@ -52,7 +59,7 @@ public class TokenNameFinderFactory extends BaseToolFactory {
}
public TokenNameFinderFactory(byte[] featureGeneratorBytes, final Map<String, Object> resources,
- SequenceCodec<String> seqCodec) {
+ SequenceCodec<String> seqCodec) {
init(featureGeneratorBytes, resources, seqCodec);
}
@@ -142,7 +149,13 @@ public class TokenNameFinderFactory extends BaseToolFactory {
AdaptiveFeatureGenerator featureGenerator = createFeatureGenerators();
if (featureGenerator == null) {
- featureGenerator = NameFinderME.createFeatureGenerator();
+ featureGenerator = new CachedFeatureGenerator(
+ new WindowFeatureGenerator(new TokenFeatureGenerator(), 2, 2),
+ new WindowFeatureGenerator(new TokenClassFeatureGenerator(true), 2, 2),
+ new OutcomePriorFeatureGenerator(),
+ new PreviousMapFeatureGenerator(),
+ new BigramNameFeatureGenerator(),
+ new SentenceFeatureGenerator(true, false));
}
return new DefaultNameContextGenerator(featureGenerator);
@@ -160,7 +173,7 @@ public class TokenNameFinderFactory extends BaseToolFactory {
public AdaptiveFeatureGenerator createFeatureGenerators() {
if (featureGeneratorBytes == null && artifactProvider != null) {
- featureGeneratorBytes = (byte[]) artifactProvider.getArtifact(
+ featureGeneratorBytes = artifactProvider.getArtifact(
TokenNameFinderModel.GENERATOR_DESCRIPTOR_ENTRY_NAME);
}
@@ -172,15 +185,12 @@ public class TokenNameFinderFactory extends BaseToolFactory {
AdaptiveFeatureGenerator generator;
try {
- generator = GeneratorFactory.create(descriptorIn, new FeatureGeneratorResourceProvider() {
-
- public Object getResource(String key) {
- if (artifactProvider != null) {
- return artifactProvider.getArtifact(key);
- }
- else {
- return resources.get(key);
- }
+ generator = GeneratorFactory.create(descriptorIn, key -> {
+ if (artifactProvider != null) {
+ return artifactProvider.getArtifact(key);
+ }
+ else {
+ return resources.get(key);
}
});
} catch (InvalidFormatException e) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
index a4780f5..ea2db50 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
@@ -23,18 +23,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
-import java.util.Collections;
import java.util.Map;
import java.util.Properties;
-
import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.model.MaxentModel;
import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.BaseToolFactory;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.SequenceCodec;
-import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
-import opennlp.tools.util.featuregen.AggregatedFeatureGenerator;
import opennlp.tools.util.featuregen.BrownCluster;
import opennlp.tools.util.featuregen.WordClusterDictionary;
import opennlp.tools.util.model.ArtifactSerializer;
@@ -58,9 +54,7 @@ public class TokenNameFinderModel extends BaseModel {
private static class ByteArraySerializer implements ArtifactSerializer<byte[]> {
- public byte[] create(InputStream in) throws IOException,
- InvalidFormatException {
-
+ public byte[] create(InputStream in) throws IOException {
return ModelUtil.read(in);
}
@@ -155,20 +149,6 @@ public class TokenNameFinderModel extends BaseModel {
checkArtifactMap();
}
- /**
- * @deprecated use getNameFinderSequenceModel instead. This method will be removed soon.
- */
- @Deprecated
- public MaxentModel getNameFinderModel() {
-
- if (artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof MaxentModel) {
- return (MaxentModel) artifactMap.get(MAXENT_MODEL_ENTRY_NAME);
- }
- else {
- return null;
- }
- }
-
public SequenceClassificationModel<String> getNameFinderSequenceModel() {
Properties manifest = (Properties) artifactMap.get(MANIFEST_ENTRY);
@@ -200,46 +180,6 @@ public class TokenNameFinderModel extends BaseModel {
return (TokenNameFinderFactory) this.toolFactory;
}
- // TODO: This should be moved to the NameFinderFactory ... !!!
- // Lets deprecate it!
-
- /**
- * Creates the {@link AdaptiveFeatureGenerator}. Usually this
- * is a set of generators contained in the {@link AggregatedFeatureGenerator}.
- *
- * Note:
- * The generators are created on every call to this method.
- *
- * @return the feature generator or null if there is no descriptor in the model
- * @deprecated use TokenNameFinderFactory.createFeatureGenerators instead!
- */
- @Deprecated
- public AdaptiveFeatureGenerator createFeatureGenerators() {
- return getFactory().createFeatureGenerators();
- }
-
- public TokenNameFinderModel updateFeatureGenerator(byte descriptor[]) {
-
- TokenNameFinderModel model;
-
- if (getNameFinderModel() != null) {
- model = new TokenNameFinderModel(getLanguage(), getNameFinderModel(), 1,
- descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(),
- getFactory().createSequenceCodec(), getFactory());
- }
- else {
- model = new TokenNameFinderModel(getLanguage(), getNameFinderSequenceModel(),
- descriptor, Collections.<String, Object>emptyMap(), Collections.<String, String>emptyMap(),
- getFactory().createSequenceCodec(), getFactory());
- }
-
- model.artifactMap.clear();
- model.artifactMap.putAll(artifactMap);
- model.artifactMap.put(GENERATOR_DESCRIPTOR_ENTRY_NAME, descriptor);
-
- return model;
- }
-
@Override
protected void createArtifactSerializers(Map<String, ArtifactSerializer> serializers) {
super.createArtifactSerializers(serializers);
@@ -276,7 +216,7 @@ public class TokenNameFinderModel extends BaseModel {
return serializers;
}
- boolean isModelValid(MaxentModel model) {
+ private boolean isModelValid(MaxentModel model) {
String outcomes[] = new String[model.getNumOutcomes()];
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerFactory.java
index 630edc4..6115994 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerFactory.java
@@ -19,7 +19,6 @@ package opennlp.tools.postag;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -27,7 +26,6 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.ml.model.AbstractModel;
import opennlp.tools.util.BaseToolFactory;
@@ -95,12 +93,12 @@ public class POSTaggerFactory extends BaseToolFactory {
}
public TagDictionary createTagDictionary(File dictionary)
- throws InvalidFormatException, FileNotFoundException, IOException {
+ throws IOException {
return createTagDictionary(new FileInputStream(dictionary));
}
public TagDictionary createTagDictionary(InputStream in)
- throws InvalidFormatException, IOException {
+ throws IOException {
return POSDictionary.create(in);
}
@@ -146,8 +144,7 @@ public class POSTaggerFactory extends BaseToolFactory {
static class POSDictionarySerializer implements ArtifactSerializer<POSDictionary> {
- public POSDictionary create(InputStream in) throws IOException,
- InvalidFormatException {
+ public POSDictionary create(InputStream in) throws IOException {
return POSDictionary.create(new UncloseableInputStream(in));
}
@@ -164,13 +161,13 @@ public class POSTaggerFactory extends BaseToolFactory {
protected void validatePOSDictionary(POSDictionary posDict,
AbstractModel posModel) throws InvalidFormatException {
- Set<String> dictTags = new HashSet<String>();
+ Set<String> dictTags = new HashSet<>();
for (String word : posDict) {
Collections.addAll(dictTags, posDict.getTags(word));
}
- Set<String> modelTags = new HashSet<String>();
+ Set<String> modelTags = new HashSet<>();
for (int i = 0; i < posModel.getNumOutcomes(); i++) {
modelTags.add(posModel.getOutcome(i));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
index cc8b0d1..e55cc17 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
@@ -18,21 +18,19 @@
package opennlp.tools.namefind;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
import java.io.InputStream;
import java.util.Collections;
-
-import opennlp.tools.ml.model.MaxentModel;
+import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.MockInputStreamFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
-
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
/**
* This is the test class for {@link NameFinderME}.
* <p>
@@ -74,7 +72,7 @@ public class NameFinderMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
- params, (byte[]) null, Collections.<String, Object>emptyMap());
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
TokenNameFinder nameFinder = new NameFinderME(nameFinderModel);
@@ -116,8 +114,6 @@ public class NameFinderMETest {
/**
* Train NamefinderME using AnnotatedSentencesWithTypes.txt with "person"
* nameType and try the model in a sample text.
- *
- * @throws Exception
*/
@Test
public void testNameFinderWithTypes() throws Exception {
@@ -137,7 +133,7 @@ public class NameFinderMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
- params, (byte[]) null, Collections.<String, Object>emptyMap());
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -167,8 +163,6 @@ public class NameFinderMETest {
/**
* Train NamefinderME using OnlyWithNames.train. The goal is to check if the model validator accepts it.
* This is related to the issue OPENNLP-9
- *
- * @throws Exception
*/
@Test
public void testOnlyWithNames() throws Exception {
@@ -186,7 +180,7 @@ public class NameFinderMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
- params, (byte[]) null, Collections.<String, Object>emptyMap());
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -206,8 +200,6 @@ public class NameFinderMETest {
/**
* Train NamefinderME using OnlyWithNamesWithTypes.train. The goal is to check if the model validator accepts it.
* This is related to the issue OPENNLP-9
- *
- * @throws Exception
*/
@Test
public void testOnlyWithNamesWithTypes() throws Exception {
@@ -225,7 +217,7 @@ public class NameFinderMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
- params, (byte[]) null, Collections.<String, Object>emptyMap());
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -247,8 +239,6 @@ public class NameFinderMETest {
/**
* Train NamefinderME using OnlyWithNames.train. The goal is to check if the model validator accepts it.
* This is related to the issue OPENNLP-9
- *
- * @throws Exception
*/
@Test
public void testOnlyWithEntitiesWithTypes() throws Exception {
@@ -266,7 +256,7 @@ public class NameFinderMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
- params, (byte[]) null, Collections.<String, Object>emptyMap());
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
@@ -283,13 +273,13 @@ public class NameFinderMETest {
}
private boolean hasOtherAsOutcome(TokenNameFinderModel nameFinderModel) {
- MaxentModel model = nameFinderModel.getNameFinderModel();
- for (int i = 0; i < model.getNumOutcomes(); i++) {
- String outcome = model.getOutcome(i);
- if (outcome.equals(NameFinderME.OTHER)) {
- return true;
- }
- }
+ SequenceClassificationModel<String> model = nameFinderModel.getNameFinderSequenceModel();
+ String[] outcomes = model.getOutcomes();
+ for (int i = 0; i < outcomes.length; i++) {
+ if (outcomes[i].equals(NameFinderME.OTHER)) {
+ return true;
+ }
+ }
return false;
}
@@ -304,8 +294,6 @@ public class NameFinderMETest {
/**
* Train NamefinderME using voa1.train with several
* nameTypes and try the model in a sample text.
- *
- * @throws Exception
*/
@Test
public void testNameFinderWithMultipleTypes() throws Exception {
@@ -323,7 +311,7 @@ public class NameFinderMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(1));
TokenNameFinderModel nameFinderModel = NameFinderME.train("en", TYPE, sampleStream,
- params, (byte[]) null, Collections.<String, Object>emptyMap());
+ params, TokenNameFinderFactory.create(null, null, Collections.emptyMap(), new BioCodec()));
NameFinderME nameFinder = new NameFinderME(nameFinderModel);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dfbf6148/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
index 22567ee..bc22aa0 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderCrossValidatorTest.java
@@ -18,15 +18,10 @@
package opennlp.tools.namefind;
import static java.nio.charset.StandardCharsets.ISO_8859_1;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
import java.io.ByteArrayOutputStream;
import java.util.Collections;
import java.util.Map;
-
-import org.junit.Test;
-
import opennlp.tools.cmdline.namefind.NameEvaluationErrorListener;
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.InputStreamFactory;
@@ -34,13 +29,17 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
public class TokenNameFinderCrossValidatorTest {
private final String TYPE = null;
@Test
- /**
+ /*
* Test that reproduces jira OPENNLP-463
*/
public void testWithNullResources() throws Exception {
@@ -67,7 +66,7 @@ public class TokenNameFinderCrossValidatorTest {
}
@Test
- /**
+ /*
* Test that tries to reproduce jira OPENNLP-466
*/
public void testWithNameEvaluationErrorListener() throws Exception {
[10/50] [abbrv] opennlp git commit: Fix hash sums of person ner and
parser test
Posted by jo...@apache.org.
Fix hash sums of person ner and parser test
See issue OPENNLP-877
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/7d1123a9
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/7d1123a9
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/7d1123a9
Branch: refs/heads/889
Commit: 7d1123a9cccaf3c22e0c2b3428c45728c3c2387e
Parents: a7826d2
Author: Joern Kottmann <ko...@gmail.com>
Authored: Mon Dec 19 16:28:35 2016 +0100
Committer: Joern Kottmann <ko...@gmail.com>
Committed: Mon Dec 19 16:28:35 2016 +0100
----------------------------------------------------------------------
.../test/java/opennlp/tools/eval/SourceForgeModelEval.java | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/7d1123a9/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
index d996afa..1e7d3ee 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/SourceForgeModelEval.java
@@ -253,7 +253,7 @@ public class SourceForgeModelEval {
TokenNameFinderModel personModel = new TokenNameFinderModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-ner-person.bin"));
- evalNameFinder(personModel, new BigInteger("260378080051855476096106859434660527393"));
+ evalNameFinder(personModel, new BigInteger("143619582249937129618340838626447763744"));
}
@Test
@@ -334,15 +334,11 @@ public class SourceForgeModelEval {
@Test
public void evalParserModel() throws IOException {
- // break input stream into sentences
- // input is tokenized already
-
ParserModel model = new ParserModel(
new File(EvalUtil.getOpennlpDataDir(), "models-sf/en-parser-chunking.bin"));
MessageDigest digest = createDigest();
-
Parser parser = ParserFactory.create(model);
try (ObjectStream<DocumentSample> lines = createLineWiseStream()) {
@@ -367,7 +363,7 @@ public class SourceForgeModelEval {
}
}
- Assert.assertEquals(new BigInteger("226003515785585284478071030961407561943"),
+ Assert.assertEquals(new BigInteger("155722144104513046994135548456420803172"),
new BigInteger(1, digest.digest()));
}
}
[34/50] [abbrv] opennlp git commit: Add japanese eos chars
Posted by jo...@apache.org.
Add japanese eos chars
Thanks to Bar Perach for providing a patch!
See issue OPENNLP-772
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/554626de
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/554626de
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/554626de
Branch: refs/heads/889
Commit: 554626de2edb69951083b847b5fd13db28bae30e
Parents: f89f3ff
Author: J�rn Kottmann <jo...@apache.org>
Authored: Tue Dec 20 23:35:48 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Tue Dec 20 23:35:48 2016 +0100
----------------------------------------------------------------------
.../java/opennlp/tools/sentdetect/lang/Factory.java | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/554626de/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java
index 1f77ade..d182b26 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/lang/Factory.java
@@ -36,14 +36,11 @@ public class Factory {
public static final char[] thEosCharacters = new char[] { ' ','\n' };
+ public static final char[] jpEosCharacters = new char[] {'\u3002', '\uff01', '\uff1f'};
+
public EndOfSentenceScanner createEndOfSentenceScanner(String languageCode) {
- if ("th".equals(languageCode)) {
- return new DefaultEndOfSentenceScanner(new char[]{' ','\n'});
- } else if("pt".equals(languageCode)) {
- return new DefaultEndOfSentenceScanner(ptEosCharacters);
- }
- return new DefaultEndOfSentenceScanner(defaultEosCharacters);
+ return new DefaultEndOfSentenceScanner(getEOSCharacters(languageCode));
}
public EndOfSentenceScanner createEndOfSentenceScanner(
@@ -76,8 +73,10 @@ public class Factory {
return thEosCharacters;
} else if ("pt".equals(languageCode)) {
return ptEosCharacters;
+ } else if ("jp".equals(languageCode)) {
+ return jpEosCharacters;
}
return defaultEosCharacters;
}
-}
\ No newline at end of file
+}
[50/50] [abbrv] opennlp git commit: Add noteworth changes for 1.7.0
to README
Posted by jo...@apache.org.
Add noteworth changes for 1.7.0 to README
See issue OPENNLP-889
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/63e969c2
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/63e969c2
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/63e969c2
Branch: refs/heads/889
Commit: 63e969c23c94d12386fa7eaebed3bc2d3bad9082
Parents: dfbf614
Author: Rodrigo Agerri <ra...@apache.org>
Authored: Wed Nov 23 13:04:49 2016 +0100
Committer: Kottmann <jo...@apache.org>
Committed: Mon Dec 26 14:28:39 2016 +0100
----------------------------------------------------------------------
opennlp-distr/README | 23 ++++++++++++-----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/63e969c2/opennlp-distr/README
----------------------------------------------------------------------
diff --git a/opennlp-distr/README b/opennlp-distr/README
index 0ecfaec..4a7a77b 100644
--- a/opennlp-distr/README
+++ b/opennlp-distr/README
@@ -22,17 +22,18 @@ removed. Now Java 1.8 is required.
Additionally the release contains the following noteworthy changes:
-- Added evalutation support to the parser and doccat components
-- Added support to Evalita 07/09, Brat and OntoNotes corpus formats
-- Now L-BFGS is stable
-- Added Snowball to the Stemmer package
-- NameFinder now supports a user defined factory
-- Added pluggable machine learning support
-- Added a lemmatizer module
-- Added Cluster, Document Begin and Clark feature generators to the Name Finder
-- Added Liblinear as a Machine Learning addon
-- Entity Linker now has a command line interface
-- Added sequence classification support
+- OpenNLP is up to 50% faster at analyzing content
+- A lot of deprecated code has been removed
+- Code base has been cleaned up
+- There is a new brat annotation service
+- Documentation was improved and extended
+- A Naive Bayesian Classifier implementation was added
+- Morofologik addon is now included
+- Added a language model component
+- Added a CLI to the lemmatizer component.
+- Added a supervised statistical lemmatizer.
+- The lemmatizer component API has been entirely rewritten. The changes in
+ the previously existing Dictionary-based lemmatizer are not backward compatible.
A detailed list of the issues related to this release can be found in the release
notes.
[16/50] [abbrv] opennlp git commit: Remove unnecessay semicolons
Posted by jo...@apache.org.
Remove unnecessay semicolons
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/9e1e77d3
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/9e1e77d3
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/9e1e77d3
Branch: refs/heads/889
Commit: 9e1e77d371f6d11432e163c0f8c171b5c77d94ea
Parents: dd483af
Author: J�rn Kottmann <jo...@apache.org>
Authored: Fri Oct 28 21:22:03 2016 +0200
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:32 2016 +0100
----------------------------------------------------------------------
.../src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java | 2 +-
.../src/main/java/opennlp/tools/ml/model/AbstractModel.java | 2 +-
.../main/java/opennlp/tools/namefind/RegexNameFinderFactory.java | 2 +-
.../src/main/java/opennlp/tools/sentdetect/SentenceSample.java | 2 +-
.../opennlp/tools/stemmer/snowball/AbstractSnowballStemmer.java | 2 +-
.../src/main/java/opennlp/tools/stemmer/snowball/Among.java | 2 +-
.../main/java/opennlp/tools/stemmer/snowball/SnowballProgram.java | 2 +-
.../src/main/java/opennlp/tools/util/AbstractEventStream.java | 2 +-
.../src/test/java/opennlp/tools/util/AbstractEventStreamTest.java | 2 +-
9 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index bfc6f9a..ee785cc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -385,7 +385,7 @@ public class ADSentenceStream extends
public void addElement(TreeElement element) {
elems.add(element);
- };
+ }
public TreeElement[] getElements() {
return elems.toArray(new TreeElement[elems.size()]);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
index 0ab1fc6..3c03c22 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
@@ -34,7 +34,7 @@ public abstract class AbstractModel implements MaxentModel {
/** Prior distribution for this model. */
protected Prior prior;
- public enum ModelType {Maxent,Perceptron,MaxentQn,NaiveBayes};
+ public enum ModelType {Maxent,Perceptron,MaxentQn,NaiveBayes}
/** The type of the model. */
protected ModelType modelType;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
index 5692d5e..0c13577 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
@@ -193,6 +193,6 @@ public class RegexNameFinderFactory {
public String getType() {
return "DEGREES_MIN_SEC_LAT_LON";
}
- };
+ }
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceSample.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceSample.java
index 81929c4..d168150 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceSample.java
@@ -43,7 +43,7 @@ public class SentenceSample {
*/
public SentenceSample(String document, Span... sentences) {
this.document = document;
- this.sentences = Collections.unmodifiableList(new ArrayList<Span>(Arrays.asList(sentences)));;
+ this.sentences = Collections.unmodifiableList(new ArrayList<Span>(Arrays.asList(sentences)));
}
public SentenceSample(Detokenizer detokenizer, String[][] sentences) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/AbstractSnowballStemmer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/AbstractSnowballStemmer.java b/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/AbstractSnowballStemmer.java
index 171a2b5..979ae74 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/AbstractSnowballStemmer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/AbstractSnowballStemmer.java
@@ -33,4 +33,4 @@ package opennlp.tools.stemmer.snowball;
abstract class AbstractSnowballStemmer extends SnowballProgram {
public abstract boolean stem();
-};
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/Among.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/Among.java b/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/Among.java
index ab71412..a93ff4d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/Among.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/Among.java
@@ -59,4 +59,4 @@ class Among {
public final int result; /* result of the lookup */
public final Method method; /* method to use if substring matches */
public final SnowballProgram methodobject; /* object to invoke method on */
-};
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/SnowballProgram.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/SnowballProgram.java b/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/SnowballProgram.java
index fcd397d..f524a5b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/SnowballProgram.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/stemmer/snowball/SnowballProgram.java
@@ -459,4 +459,4 @@ extern void debug(struct SN_env * z, int number, int line_count)
}
*/
-};
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
index 1e48b25..1a61d87 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
@@ -28,7 +28,7 @@ public abstract class AbstractEventStream<T> implements ObjectStream<Event> {
private ObjectStream<T> samples;
- private Iterator<Event> events = Collections.<Event>emptyList().iterator();;
+ private Iterator<Event> events = Collections.<Event>emptyList().iterator();
/**
* Initializes the current instance with a sample {@link Iterator}.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9e1e77d3/opennlp-tools/src/test/java/opennlp/tools/util/AbstractEventStreamTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/AbstractEventStreamTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/AbstractEventStreamTest.java
index f0d5e6d..ab44a4c 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/AbstractEventStreamTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/AbstractEventStreamTest.java
@@ -39,7 +39,7 @@ public class AbstractEventStreamTest {
private enum RESULT {
EVENTS,
EMPTY
- };
+ }
/**
[41/50] [abbrv] opennlp git commit: OPENNLP-899: Replace deprecated
code from Tokenizer Trainer
Posted by jo...@apache.org.
OPENNLP-899: Replace deprecated code from Tokenizer Trainer
This closes #13
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/927ee0fc
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/927ee0fc
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/927ee0fc
Branch: refs/heads/889
Commit: 927ee0fc7f47c7c6ccbd72a184aaa6215d77943c
Parents: bbda5de
Author: smarthi <sm...@apache.org>
Authored: Wed Dec 21 14:35:49 2016 -0500
Committer: Kottmann <jo...@apache.org>
Committed: Thu Dec 22 16:25:32 2016 +0100
----------------------------------------------------------------------
.../tools/tokenize/TokenizerCrossValidator.java | 4 +---
.../opennlp/tools/tokenize/TokenizerFactory.java | 2 +-
.../java/opennlp/tools/tokenize/TokenizerME.java | 8 ++++----
.../opennlp/tools/tokenize/TokenizerModel.java | 17 ++++++-----------
.../opennlp/tools/eval/ArvoresDeitadasEval.java | 2 +-
.../opennlp/tools/tokenize/TokenizerTestUtil.java | 11 +++++------
.../opennlp/uima/tokenize/TokenizerTrainer.java | 6 +++++-
7 files changed, 23 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
index 811165c..3ca3c1d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
@@ -94,9 +94,7 @@ public class TokenizerCrossValidator {
partitioner.next();
// Maybe throws IOException if temporary file handling fails ...
- TokenizerModel model;
-
- model = TokenizerME.train(trainingSampleStream, this.factory, params);
+ TokenizerModel model = TokenizerME.train(trainingSampleStream, this.factory, params);
TokenizerEvaluator evaluator = new TokenizerEvaluator(new TokenizerME(model), listeners);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
index 4c67ce1..f9e789a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
@@ -37,7 +37,7 @@ public class TokenizerFactory extends BaseToolFactory {
private String languageCode;
private Dictionary abbreviationDictionary;
- private Boolean useAlphaNumericOptimization = null;
+ private Boolean useAlphaNumericOptimization;
private Pattern alphaNumericPattern;
private static final String ABBREVIATIONS_ENTRY_NAME = "abbreviations.dictionary";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
index 5412c28..4c4c638 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
@@ -52,7 +52,7 @@ import opennlp.tools.util.model.ModelUtil;
* must be instantiated which can share one <code>TokenizerModel</code> instance
* to safe memory.
* <p>
- * To train a new model {{@link #train(String, ObjectStream, boolean, TrainingParameters)} method
+ * To train a new model {{@link #train(ObjectStream, TokenizerFactory, TrainingParameters)} method
* can be used.
* <p>
* Sample usage:
@@ -250,8 +250,7 @@ public class TokenizerME extends AbstractTokenizer {
MaxentModel maxentModel = trainer.train(eventStream);
- return new TokenizerModel(maxentModel, manifestInfoEntries,
- factory);
+ return new TokenizerModel(maxentModel, manifestInfoEntries, factory);
}
/**
@@ -338,7 +337,8 @@ public class TokenizerME extends AbstractTokenizer {
*/
public static TokenizerModel train(String languageCode, ObjectStream<TokenSample> samples,
boolean useAlphaNumericOptimization) throws IOException {
- return train(languageCode, samples, useAlphaNumericOptimization, ModelUtil.createDefaultTrainingParameters());
+ return train(samples, TokenizerFactory.create(null, languageCode, null, useAlphaNumericOptimization, null),
+ ModelUtil.createDefaultTrainingParameters());
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
index 1af60f4..e63b946 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
@@ -124,7 +124,7 @@ public final class TokenizerModel extends BaseModel {
* @throws IOException if reading from the stream fails in anyway
* @throws InvalidFormatException if the stream doesn't have the expected format
*/
- public TokenizerModel(InputStream in) throws IOException, InvalidFormatException {
+ public TokenizerModel(InputStream in) throws IOException {
super(COMPONENT_NAME, in);
}
@@ -134,9 +134,8 @@ public final class TokenizerModel extends BaseModel {
* @param modelFile the file containing the tokenizer model
*
* @throws IOException if reading from the stream fails in anyway
- * @throws InvalidFormatException if the stream doesn't have the expected format
*/
- public TokenizerModel(File modelFile) throws IOException, InvalidFormatException {
+ public TokenizerModel(File modelFile) throws IOException {
super(COMPONENT_NAME, modelFile);
}
@@ -146,9 +145,8 @@ public final class TokenizerModel extends BaseModel {
* @param modelURL the URL pointing to the tokenizer model
*
* @throws IOException if reading from the stream fails in anyway
- * @throws InvalidFormatException if the stream doesn't have the expected format
*/
- public TokenizerModel(URL modelURL) throws IOException, InvalidFormatException {
+ public TokenizerModel(URL modelURL) throws IOException {
super(COMPONENT_NAME, modelURL);
}
@@ -196,10 +194,7 @@ public final class TokenizerModel extends BaseModel {
}
public boolean useAlphaNumericOptimization() {
- if (getFactory() != null) {
- return getFactory().isUseAlphaNumericOptmization();
- }
- return false;
+ return getFactory() != null && getFactory().isUseAlphaNumericOptmization();
}
public static void main(String[] args) throws IOException {
@@ -224,8 +219,8 @@ public final class TokenizerModel extends BaseModel {
AbstractModel model = new BinaryGISModelReader(new DataInputStream(
new FileInputStream(modelName))).getModel();
- TokenizerModel packageModel = new TokenizerModel(languageCode, model,
- alphaNumericOptimization);
+ TokenizerModel packageModel = new TokenizerModel(model, null,
+ TokenizerFactory.create(null, languageCode, null, alphaNumericOptimization, null));
OutputStream out = null;
try {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
index 35f0e00..33d6ffe 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/ArvoresDeitadasEval.java
@@ -72,7 +72,7 @@ public class ArvoresDeitadasEval {
private static final String LANG = "pt";
- private static final TrainingParameters getPerceptronZeroCutoff() {
+ private static TrainingParameters getPerceptronZeroCutoff() {
TrainingParameters params = ModelUtil.createDefaultTrainingParameters();
params.put(TrainingParameters.ALGORITHM_PARAM,
PerceptronTrainer.PERCEPTRON_VALUE);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
index f8eb85b..ffe5101 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerTestUtil.java
@@ -19,11 +19,10 @@
package opennlp.tools.tokenize;
import static java.nio.charset.StandardCharsets.UTF_8;
+
import java.io.IOException;
-import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
-
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.util.CollectionObjectStream;
import opennlp.tools.util.InputStreamFactory;
@@ -38,7 +37,7 @@ import opennlp.tools.util.TrainingParameters;
public class TokenizerTestUtil {
static TokenizerModel createSimpleMaxentTokenModel() throws IOException {
- List<TokenSample> samples = new ArrayList<TokenSample>();
+ List<TokenSample> samples = new ArrayList<>();
samples.add(new TokenSample("year", new Span[]{new Span(0, 4)}));
samples.add(new TokenSample("year,", new Span[]{
@@ -59,8 +58,8 @@ public class TokenizerTestUtil {
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
mlParams.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(0));
- return TokenizerME.train("en", new CollectionObjectStream<TokenSample>(samples), true,
- mlParams);
+ return TokenizerME.train(new CollectionObjectStream<>(samples),
+ TokenizerFactory.create(null, "en", null, true, null), mlParams);
}
static TokenizerModel createMaxentTokenModel() throws IOException {
@@ -75,7 +74,7 @@ public class TokenizerTestUtil {
mlParams.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
mlParams.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(0));
- return TokenizerME.train("en", samples, true, mlParams);
+ return TokenizerME.train(samples, TokenizerFactory.create(null, "en", null, true, null), mlParams);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/927ee0fc/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
index 66d1dfa..2b36051 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/tokenize/TokenizerTrainer.java
@@ -32,6 +32,7 @@ import java.util.List;
import opennlp.tools.ml.maxent.GIS;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.tokenize.TokenSampleStream;
+import opennlp.tools.tokenize.TokenizerFactory;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.InputStreamFactory;
@@ -40,6 +41,7 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.ObjectStreamUtils;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
+import opennlp.tools.util.model.ModelUtil;
import opennlp.uima.util.CasConsumerUtil;
import opennlp.uima.util.ContainingConstraint;
import opennlp.uima.util.OpennlpUtil;
@@ -257,7 +259,9 @@ public final class TokenizerTrainer extends CasConsumer_ImplBase {
samples = new SampleTraceStream<>(samples, samplesOut);
}
- tokenModel = TokenizerME.train(language, samples, isSkipAlphaNumerics);
+ tokenModel = TokenizerME.train(samples,
+ TokenizerFactory.create(null, language, null, isSkipAlphaNumerics, null),
+ ModelUtil.createDefaultTrainingParameters());
// dereference to allow garbage collection
tokenSamples = null;
[26/50] [abbrv] opennlp git commit: Remove redundant variable
initialization
Posted by jo...@apache.org.
Remove redundant variable initialization
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/06371807
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/06371807
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/06371807
Branch: refs/heads/889
Commit: 06371807f421926eefb99b0ce15dd552066b0cab
Parents: eb93f64
Author: J�rn Kottmann <jo...@apache.org>
Authored: Mon Nov 7 18:19:17 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:39:08 2016 +0100
----------------------------------------------------------------------
.../tools/cmdline/GenerateManualTool.java | 2 +-
.../tools/cmdline/parser/ParserTrainerTool.java | 2 +-
.../entitylinker/EntityLinkerProperties.java | 5 +---
.../tools/formats/ad/ADNameSampleStream.java | 3 +--
.../formats/brat/AnnotationConfiguration.java | 2 +-
.../tools/lemmatizer/DictionaryLemmatizer.java | 2 +-
.../opennlp/tools/ml/AbstractEventTrainer.java | 2 +-
.../opennlp/tools/ml/maxent/GISTrainer.java | 6 ++---
.../tools/ml/maxent/quasinewton/QNTrainer.java | 4 +--
.../tools/namefind/DictionaryNameFinder.java | 2 +-
.../tools/namefind/TokenNameFinderFactory.java | 2 +-
.../tools/parser/AbstractBottomUpParser.java | 6 ++---
.../parser/chunking/BuildContextGenerator.java | 19 +++++++------
.../opennlp/tools/parser/chunking/Parser.java | 2 +-
.../treeinsert/AttachContextGenerator.java | 9 +++----
.../treeinsert/BuildContextGenerator.java | 28 +++++++++-----------
.../treeinsert/CheckContextGenerator.java | 6 ++---
.../sentdetect/SentenceDetectorFactory.java | 2 +-
.../opennlp/tools/tokenize/TokenSample.java | 4 +--
.../tools/tokenize/TokenizerFactory.java | 2 +-
.../opennlp/tools/util/AbstractEventStream.java | 2 +-
.../opennlp/tools/util/BaseToolFactory.java | 2 +-
.../opennlp/tools/util/model/BaseModel.java | 7 ++---
23 files changed, 53 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
index b42f69c..b2a9c42 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
@@ -239,7 +239,7 @@ public class GenerateManualTool {
*/
private static String splitLongLines(String stringBlock) {
StringBuilder sb = new StringBuilder();
- String line = null;
+ String line;
try {
BufferedReader reader = new BufferedReader(new StringReader(stringBlock));
while ((line = reader.readLine()) != null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
index 5ea7ec4..10cf7c4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
@@ -83,7 +83,7 @@ public final class ParserTrainerTool extends AbstractTrainerTool<Parse, TrainerT
static HeadRules creaeHeadRules(TrainerToolParams params) throws IOException {
- ArtifactSerializer headRulesSerializer = null;
+ ArtifactSerializer headRulesSerializer;
if (params.getHeadRulesSerializerImpl() != null) {
headRulesSerializer = ExtensionLoader.instantiateExtension(ArtifactSerializer.class,
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerProperties.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerProperties.java b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerProperties.java
index 31e6486..bd7b6f6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerProperties.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/entitylinker/EntityLinkerProperties.java
@@ -76,13 +76,10 @@ public class EntityLinkerProperties {
*/
public String getProperty(String key, String defaultValue) throws IOException {
- String propVal = defaultValue;
-
if (props != null) {
- propVal = props.getProperty(key, defaultValue);
+ return props.getProperty(key, defaultValue);
} else {
throw new IOException("EntityLinkerProperties was not successfully initialized");
}
- return propVal;
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
index 22b3efb..5e131f2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
@@ -347,10 +347,9 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
// a NER.
// we check if it is true, and expand the last NER
int lastIndex = names.size() - 1;
- Span last = null;
boolean error = false;
if (names.size() > 0) {
- last = names.get(lastIndex);
+ Span last = names.get(lastIndex);
if (last.getEnd() == sentence.size() - 1) {
names.set(lastIndex, new Span(last.getStart(), sentence.size(),
last.getType()));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
index 505ba4e..723ddbc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/AnnotationConfiguration.java
@@ -56,7 +56,7 @@ public class AnnotationConfiguration {
BufferedReader reader = new BufferedReader(new InputStreamReader(in, Charset.forName("UTF-8")));
// Note: This only supports entities and relations section
- String line = null;
+ String line;
String sectionType = null;
while ((line = reader.readLine())!= null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
index 815f35e..0ae8e37 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
@@ -100,7 +100,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
* @return the lemma
*/
public String apply(final String word, final String postag) {
- String lemma = null;
+ String lemma;
final List<String> keys = this.getDictKeys(word, postag);
// lookup lemma as value of the map
final String keyValue = this.dictMap.get(keys);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/ml/AbstractEventTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/AbstractEventTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/AbstractEventTrainer.java
index cd1aa42..ae0c3b0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/AbstractEventTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/AbstractEventTrainer.java
@@ -66,7 +66,7 @@ public abstract class AbstractEventTrainer extends AbstractTrainer implements
int cutoff = getCutoff();
boolean sortAndMerge = isSortAndMerge();
- DataIndexer indexer = null;
+ DataIndexer indexer;
if (DATA_INDEXER_ONE_PASS_VALUE.equals(dataIndexerName)) {
indexer = new OnePassDataIndexer(events, cutoff, sortAndMerge);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
index bd6603e..0527979 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
@@ -340,7 +340,7 @@ class GISTrainer {
for (int oi = 0; oi < numOutcomes; oi++) {
allOutcomesPattern[oi] = oi;
}
- int numActiveOutcomes = 0;
+ int numActiveOutcomes;
for (int pi = 0; pi < numPreds; pi++) {
numActiveOutcomes = 0;
if (useSimpleSmoothing) {
@@ -404,7 +404,7 @@ class GISTrainer {
/* Estimate and return the model parameters. */
private void findParameters(int iterations, double correctionConstant) {
double prevLL = 0.0;
- double currLL = 0.0;
+ double currLL;
display("Performing " + iterations + " iterations.\n");
for (int i = 1; i <= iterations; i++) {
if (i < 10)
@@ -569,7 +569,7 @@ class GISTrainer {
}
for (Future<?> future : futures) {
- ModelExpactationComputeTask finishedTask = null;
+ ModelExpactationComputeTask finishedTask;
try {
finishedTask = (ModelExpactationComputeTask) future.get();
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNTrainer.java
index b8deb52..dbcc331 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/quasinewton/QNTrainer.java
@@ -156,7 +156,7 @@ public class QNTrainer extends AbstractEventTrainer {
public QNModel trainModel(int iterations, DataIndexer indexer) {
// Train model's parameters
- Function objectiveFunction = null;
+ Function objectiveFunction;
if (threads == 1) {
System.out.println("Computing model parameters ...");
objectiveFunction = new NegLogLikelihood(indexer);
@@ -237,4 +237,4 @@ public class QNTrainer extends AbstractEventTrainer {
return (double) nCorrect / nTotalEvents;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
index b9e1f17..a8d249f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
@@ -67,7 +67,7 @@ public class DictionaryNameFinder implements TokenNameFinder {
for (int offsetFrom = 0; offsetFrom < textTokenized.length; offsetFrom++) {
Span nameFound = null;
- String tokensSearching[] = new String[] {};
+ String tokensSearching[];
for (int offsetTo = offsetFrom; offsetTo < textTokenized.length; offsetTo++) {
int lengthSearching = offsetTo - offsetFrom + 1;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
index 3864c84..90381ff 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderFactory.java
@@ -170,7 +170,7 @@ public class TokenNameFinderFactory extends BaseToolFactory {
InputStream descriptorIn = new ByteArrayInputStream(featureGeneratorBytes);
- AdaptiveFeatureGenerator generator = null;
+ AdaptiveFeatureGenerator generator;
try {
generator = GeneratorFactory.create(descriptorIn, new FeatureGeneratorResourceProvider() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
index 0a05d61..3fabfd2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
@@ -219,7 +219,7 @@ public abstract class AbstractBottomUpParser implements Parser {
public static Parse[] collapsePunctuation(Parse[] chunks, Set<String> punctSet) {
List<Parse> collapsedParses = new ArrayList<Parse>(chunks.length);
int lastNonPunct = -1;
- int nextNonPunct = -1;
+ int nextNonPunct;
for (int ci=0,cn=chunks.length;ci<cn;ci++) {
if (punctSet.contains(chunks[ci].getType())) {
if (lastNonPunct >= 0) {
@@ -383,9 +383,9 @@ public abstract class AbstractBottomUpParser implements Parser {
String words[] = new String[children.length];
String ptags[] = new String[words.length];
double probs[] = new double[words.length];
- Parse sp = null;
+
for (int i = 0, il = children.length; i < il; i++) {
- sp = children[i];
+ Parse sp = children[i];
words[i] = sp.getHead().getCoveredText();
ptags[i] = sp.getType();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
index 081ef9a..42d7c07 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
@@ -73,32 +73,31 @@ public class BuildContextGenerator extends AbstractContextGenerator {
// cons(-2), cons(-1), cons(0), cons(1), cons(2)
// cons(-2)
- Parse p_2 = null;
- Parse p_1 = null;
- Parse p0 = null;
- Parse p1 = null;
- Parse p2 = null;
- Collection<Parse> punct1s = null;
Collection<Parse> punct2s = null;
- Collection<Parse> punct_1s = null;
Collection<Parse> punct_2s = null;
+ Parse p_2 = null;
if (index - 2 >= 0) {
p_2 = constituents[index - 2];
}
+
+ Parse p_1 = null;
if (index - 1 >= 0) {
p_1 = constituents[index - 1];
punct_2s = p_1.getPreviousPunctuationSet();
}
- p0 = constituents[index];
- punct_1s=p0.getPreviousPunctuationSet();
- punct1s=p0.getNextPunctuationSet();
+ Parse p0 = constituents[index];
+ Collection<Parse> punct_1s = p0.getPreviousPunctuationSet();
+ Collection<Parse> punct1s = p0.getNextPunctuationSet();
+ Parse p1 = null;
if (index + 1 < ps) {
p1 = constituents[index + 1];
punct2s = p1.getNextPunctuationSet();
}
+
+ Parse p2 = null;
if (index + 2 < ps) {
p2 = constituents[index + 2];
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
index 05314ef..062e7f2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
@@ -206,7 +206,7 @@ public class Parser extends AbstractBottomUpParser {
//String[] context = checkContextGenerator.getContext(newParse1.getChildren(), lastStartType, lastStartIndex, advanceNodeIndex);
checkModel.eval(checkContextGenerator.getContext(collapsePunctuation(newParse1.getChildren(),punctSet), lastStartType, lastStartIndex, advanceNodeIndex), cprobs);
//System.out.println("check "+lastStartType+" "+cprobs[completeIndex]+" "+cprobs[incompleteIndex]+" "+tag+" "+java.util.Arrays.asList(context));
- Parse newParse2 = newParse1;
+ Parse newParse2;
if (cprobs[completeIndex] > q) { //make sure a reduce is likely
newParse2 = (Parse) newParse1.clone();
if (createDerivationString) newParse2.getDerivation().append(1).append(".");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
index 7ba8a90..9bce4cc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
@@ -77,12 +77,9 @@ public class AttachContextGenerator extends AbstractContextGenerator {
p1 = constituents[index+1];
}
- Collection<Parse> punct1s = null;
- Collection<Parse> punct_1s = null;
- Collection<Parse> punct_1fs = null;
- punct_1fs = fn.getPreviousPunctuationSet();
- punct_1s=p0.getPreviousPunctuationSet();
- punct1s=p0.getNextPunctuationSet();
+ Collection<Parse> punct_1fs = fn.getPreviousPunctuationSet();
+ Collection<Parse> punct_1s = p0.getPreviousPunctuationSet();
+ Collection<Parse> punct1s = p0.getNextPunctuationSet();
String consfp = cons(fp,-3);
String consf = cons(fn,-2);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
index 0d91612..8677171 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
@@ -55,30 +55,27 @@ public class BuildContextGenerator extends AbstractContextGenerator {
* @return the contexts/features for the decision to build a new constituent.
*/
public String[] getContext(Parse[] constituents, int index) {
- Parse p_2 = null;
- Parse p_1 = null;
- Parse p0 = null;
- Parse p1 = null;
- Parse p2 = null;
int ps = constituents.length;
- p0 = constituents[index];
+ Parse p0 = constituents[index];
+
+ Parse p1 = null;
if (index + 1 < ps) {
p1 = constituents[index + 1];
}
+
+ Parse p2 = null;
if (index +2 < ps) {
p2 = constituents[index + 2];
}
- Collection<Parse> punct1s = null;
- Collection<Parse> punct_1s = null;
- Collection<Parse> punct2s = null;
- Collection<Parse> punct_2s = null;
- punct_1s=p0.getPreviousPunctuationSet();
- punct1s=p0.getNextPunctuationSet();
+ Collection<Parse> punct_1s = p0.getPreviousPunctuationSet();
+ Collection<Parse> punct1s = p0.getNextPunctuationSet();
+
+ Collection<Parse> punct2s = null;
if (p1 != null) {
- punct2s=p1.getNextPunctuationSet();
+ punct2s = p1.getNextPunctuationSet();
}
@@ -92,9 +89,10 @@ public class BuildContextGenerator extends AbstractContextGenerator {
rf = Parser.getRightFrontier(constituents[0], emptyPunctSet);
}
getFrontierNodes(rf,leftNodes);
- p_1 = leftNodes[0];
- p_2 = leftNodes[1];
+ Parse p_1 = leftNodes[0];
+ Parse p_2 = leftNodes[1];
+ Collection<Parse> punct_2s = null;
if (p_1 != null) {
punct_2s = p_1.getPreviousPunctuationSet();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
index 199b4a2..694d65d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
@@ -59,8 +59,6 @@ public class CheckContextGenerator extends AbstractContextGenerator {
Parse p1 = null;
Parse p2 = null;
- Parse p_1 = null;
- Parse p_2 = null;
Collection<Parse> p1s = constituents[index].getNextPunctuationSet();
Collection<Parse> p2s = null;
Collection<Parse> p_1s = constituents[index].getPreviousPunctuationSet();
@@ -86,8 +84,8 @@ public class CheckContextGenerator extends AbstractContextGenerator {
}
getFrontierNodes(rf,leftNodes);
- p_1 = leftNodes[0];
- p_2 = leftNodes[1];
+ Parse p_1 = leftNodes[0];
+ Parse p_2 = leftNodes[1];
int ps = constituents.length;
if (p_1 != null) {
p_2s = p_1.getPreviousPunctuationSet();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
index c6d02cb..9ef3e78 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
@@ -190,7 +190,7 @@ public class SentenceDetectorFactory extends BaseToolFactory {
public SDContextGenerator getSDContextGenerator() {
Factory f = new Factory();
char[] eosChars = getEOSCharacters();
- Set<String> abbs = null;
+ Set<String> abbs;
Dictionary abbDict = getAbbreviationDictionary();
if (abbDict != null) {
abbs = abbDict.asStringSet();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenSample.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenSample.java
index 84021da..1a40dd5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenSample.java
@@ -130,7 +130,7 @@ public class TokenSample {
// and this token insert the separator chars
// otherwise insert a space
- String separator = "";
+ String separator;
if (lastEndIndex == token.getStart())
separator = separatorChars;
else
@@ -181,7 +181,7 @@ public class TokenSample {
boolean wasTokenReplaced = false;
int tokStart = 0;
- int tokEnd = -1;
+ int tokEnd;
while ((tokEnd = whitespaceToken.indexOf(separatorChars, tokStart)) > -1) {
String token = whitespaceToken.substring(tokStart, tokEnd);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
index e1095e3..4c67ce1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
@@ -228,7 +228,7 @@ public class TokenizerFactory extends BaseToolFactory {
*/
public TokenContextGenerator getContextGenerator() {
Factory f = new Factory();
- Set<String> abbs = null;
+ Set<String> abbs;
Dictionary abbDict = getAbbreviationDictionary();
if (abbDict != null) {
abbs = abbDict.asStringSet();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
index 1a61d87..c636378 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/AbstractEventStream.java
@@ -57,7 +57,7 @@ public abstract class AbstractEventStream<T> implements ObjectStream<Event> {
return events.next();
}
else {
- T sample = null;
+ T sample;
while (!events.hasNext() && (sample = samples.read()) != null) {
events = createEvents(sample);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java b/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
index 69f13dc..873c2c4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
@@ -101,7 +101,7 @@ public abstract class BaseToolFactory {
public static BaseToolFactory create(String subclassName,
ArtifactProvider artifactProvider) throws InvalidFormatException {
- BaseToolFactory theFactory = null;
+ BaseToolFactory theFactory;
try {
// load the ToolFactory using the default constructor
http://git-wip-us.apache.org/repos/asf/opennlp/blob/06371807/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
index 771958f..9997a82 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
@@ -347,14 +347,11 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
}
protected ArtifactSerializer getArtifactSerializer(String resourceName) {
- String extension = null;
try {
- extension = getEntryExtension(resourceName);
+ return artifactSerializers.get(getEntryExtension(resourceName));
} catch (InvalidFormatException e) {
throw new IllegalStateException(e);
}
-
- return artifactSerializers.get(extension);
}
protected static Map<String, ArtifactSerializer> createArtifactSerializers() {
@@ -621,7 +618,7 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
ByteArrayOutputStream output = new ByteArrayOutputStream();
byte[] buffer = new byte[1024 * 4];
int count = 0;
- int n = 0;
+ int n;
while (-1 != (n = input.read(buffer))) {
output.write(buffer, 0, n);
count += n;
[02/50] [abbrv] opennlp git commit: Make all models Serializable
Posted by jo...@apache.org.
Make all models Serializable
Thanks to Tristan Nixon for providing a patch.
See issue OPENNLP-776
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c9f2943c
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c9f2943c
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c9f2943c
Branch: refs/heads/889
Commit: c9f2943c76fcc3191639400dbf8f383869ef814b
Parents: c28a542
Author: J�rn Kottmann <jo...@apache.org>
Authored: Fri Oct 28 18:00:10 2016 +0200
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Tue Nov 22 01:20:13 2016 +0100
----------------------------------------------------------------------
.../opennlp/tools/util/model/BaseModel.java | 36 ++++++++++++++++----
1 file changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c9f2943c/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
index 989c80b..4a40d15 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
@@ -24,7 +24,10 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.io.OutputStream;
+import java.io.Serializable;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
@@ -45,7 +48,7 @@ import opennlp.tools.util.ext.ExtensionLoader;
* TODO:
* Provide sub classes access to serializers already in constructor
*/
-public abstract class BaseModel implements ArtifactProvider {
+public abstract class BaseModel implements ArtifactProvider, Serializable {
private static int MODEL_BUFFER_SIZE_LIMIT = Integer.MAX_VALUE;
@@ -64,19 +67,18 @@ public abstract class BaseModel implements ArtifactProvider {
private static String SERIALIZER_CLASS_NAME_PREFIX = "serializer-class-";
- private Map<String, ArtifactSerializer> artifactSerializers =
- new HashMap<String, ArtifactSerializer>();
+ private Map<String, ArtifactSerializer> artifactSerializers = new HashMap<>();
- protected final Map<String, Object> artifactMap = new HashMap<String, Object>();
+ protected Map<String, Object> artifactMap = new HashMap<>();
protected BaseToolFactory toolFactory;
- private final String componentName;
+ private String componentName;
private boolean subclassSerializersInitiated = false;
private boolean finishedLoadingArtifacts = false;
- private final boolean isLoadedFromSerialized;
+ private boolean isLoadedFromSerialized;
private BaseModel(String componentName, boolean isLoadedFromSerialized) {
this.isLoadedFromSerialized = isLoadedFromSerialized;
@@ -632,4 +634,26 @@ public abstract class BaseModel implements ArtifactProvider {
public boolean isLoadedFromSerialized() {
return isLoadedFromSerialized;
}
+
+ // These methods are required to serialize/deserialize the model because
+ // many of the included objects in this model are not Serializable.
+ // An alternative to this solution is to make all included objects
+ // Serializable and remove the writeObject and readObject methods.
+ // This will allow the usage of final for fields that should not change.
+
+ private void writeObject(ObjectOutputStream out) throws IOException {
+ out.writeUTF(componentName);
+ this.serialize(out);
+ }
+
+ private void readObject(final ObjectInputStream in) throws IOException {
+
+ isLoadedFromSerialized = true;
+ artifactSerializers = new HashMap<>();
+ artifactMap = new HashMap<>();
+
+ componentName = in.readUTF();
+
+ this.loadModel(in);
+ }
}
[12/50] [abbrv] opennlp git commit: opennlp cmd adds all jars in lib
to classpath
Posted by jo...@apache.org.
opennlp cmd adds all jars in lib to classpath
See issue OPENNLP-893
This closes #7
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/34af4aec
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/34af4aec
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/34af4aec
Branch: refs/heads/889
Commit: 34af4aec2d7f3f5513de09a00b2a730c7570dc6c
Parents: 4da7f4c
Author: Dr. Llama <pe...@gmail.com>
Authored: Thu Dec 8 00:13:09 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 22:38:54 2016 +0100
----------------------------------------------------------------------
opennlp-distr/src/main/bin/opennlp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/34af4aec/opennlp-distr/src/main/bin/opennlp
----------------------------------------------------------------------
diff --git a/opennlp-distr/src/main/bin/opennlp b/opennlp-distr/src/main/bin/opennlp
index 6e2fdd2..d48c622 100755
--- a/opennlp-distr/src/main/bin/opennlp
+++ b/opennlp-distr/src/main/bin/opennlp
@@ -32,4 +32,4 @@ fi
# Might fail if $0 is a link
OPENNLP_HOME=`dirname "$0"`/..
-$JAVACMD -Xmx1024m -jar $OPENNLP_HOME/lib/opennlp-tools-*.jar $@
+$JAVACMD -Xmx1024m -cp $(echo $OPENNLP_HOME/lib/*.jar | tr ' ' ':') opennlp.tools.cmdline.CLI $@
[07/50] [abbrv] opennlp git commit: OPENNLP-891: Making the
TokenClassFeatureGenerator configurable.
Posted by jo...@apache.org.
OPENNLP-891: Making the TokenClassFeatureGenerator configurable.
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/08065623
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/08065623
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/08065623
Branch: refs/heads/889
Commit: 080656237bb56d52ca97f4da4171b0ce1ec42e4a
Parents: d62cc41
Author: jzonthemtn <je...@mtnfog.com>
Authored: Mon Dec 12 19:55:57 2016 -0500
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Wed Dec 14 00:42:24 2016 +0100
----------------------------------------------------------------------
.../tools/util/featuregen/GeneratorFactory.java | 16 +++++++++++--
.../featuregen/TokenClassFeatureGenerator.java | 4 ++--
.../util/featuregen/GeneratorFactoryTest.java | 17 ++++++++++++++
.../TestTokenClassFeatureGeneratorConfig.xml | 24 ++++++++++++++++++++
4 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/08065623/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
index d943381..f6cf702 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/GeneratorFactory.java
@@ -427,8 +427,20 @@ public class GeneratorFactory {
public AdaptiveFeatureGenerator create(Element generatorElement,
FeatureGeneratorResourceProvider resourceManager) {
- // TODO: Make it configurable ...
- return new TokenClassFeatureGenerator(true);
+
+ String attribute = generatorElement.getAttribute("wordAndClass");
+
+ // Default to true.
+ boolean generateWordAndClassFeature = true;
+
+ if(attribute != "") {
+ // Anything other than "true" sets it to false.
+ if(!"true".equalsIgnoreCase(attribute)) {
+ generateWordAndClassFeature = false;
+ }
+ }
+
+ return new TokenClassFeatureGenerator(generateWordAndClassFeature);
}
static void register(Map<String, XmlFeatureGeneratorFactory> factoryMap) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/08065623/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
index 1a810e0..4aad40c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
@@ -36,8 +36,8 @@ public class TokenClassFeatureGenerator extends FeatureGeneratorAdapter {
this(false);
}
- public TokenClassFeatureGenerator(boolean genearteWordAndClassFeature) {
- this.generateWordAndClassFeature = genearteWordAndClassFeature;
+ public TokenClassFeatureGenerator(boolean generateWordAndClassFeature) {
+ this.generateWordAndClassFeature = generateWordAndClassFeature;
}
public void createFeatures(List<String> features, String[] tokens, int index, String[] preds) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/08065623/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/GeneratorFactoryTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/GeneratorFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/GeneratorFactoryTest.java
index b0a1500..9a28588 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/GeneratorFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/GeneratorFactoryTest.java
@@ -35,6 +35,23 @@ import opennlp.tools.util.model.ArtifactSerializer;
import org.junit.Test;
public class GeneratorFactoryTest {
+
+ @Test
+ public void testCreationWithTokenClassFeatureGenerator() throws Exception {
+ InputStream generatorDescriptorIn = getClass().getResourceAsStream(
+ "/opennlp/tools/util/featuregen/TestTokenClassFeatureGeneratorConfig.xml");
+
+ // If this fails the generator descriptor could not be found
+ // at the expected location
+ assertNotNull(generatorDescriptorIn);
+
+ AggregatedFeatureGenerator aggregatedGenerator =
+ (AggregatedFeatureGenerator) GeneratorFactory.create(generatorDescriptorIn, null);
+
+ assertEquals(1, aggregatedGenerator.getGenerators().size());
+ assertEquals(TokenClassFeatureGenerator.class.getName(), aggregatedGenerator.getGenerators().iterator().next().getClass().getName());
+
+ }
@Test
public void testCreationWihtSimpleDescriptor() throws Exception {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/08065623/opennlp-tools/src/test/resources/opennlp/tools/util/featuregen/TestTokenClassFeatureGeneratorConfig.xml
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/util/featuregen/TestTokenClassFeatureGeneratorConfig.xml b/opennlp-tools/src/test/resources/opennlp/tools/util/featuregen/TestTokenClassFeatureGeneratorConfig.xml
new file mode 100644
index 0000000..e384a7e
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/util/featuregen/TestTokenClassFeatureGeneratorConfig.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<generators>
+ <tokenclass wordAndClass="true"/>
+</generators>
\ No newline at end of file
[06/50] [abbrv] opennlp git commit: Revert "Remove deprecated API
from the Parser"
Posted by jo...@apache.org.
Revert "Remove deprecated API from the Parser"
That should not have been merged, caused compile errors.
This reverts commit 9c140512d9caadda9ea469b679766870304ce947.
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/d62cc416
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/d62cc416
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/d62cc416
Branch: refs/heads/889
Commit: d62cc416b5fb6db89b624c3b5203711ad597ebe4
Parents: 9c14051
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Dec 14 00:39:34 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Wed Dec 14 00:39:34 2016 +0100
----------------------------------------------------------------------
.../opennlp/tools/parser/chunking/Parser.java | 32 ++++++++++++++++++++
1 file changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d62cc416/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
index 77a030f..044477b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
@@ -248,6 +248,15 @@ public class Parser extends AbstractBottomUpParser {
return newParses;
}
+ /**
+ * @deprecated Please do not use anymore, use the ObjectStream train methods instead! This method
+ * will be removed soon.
+ */
+ @Deprecated
+ public static AbstractModel train(ObjectStream<Event> es, int iterations, int cut) throws java.io.IOException {
+ return opennlp.tools.ml.maxent.GIS.trainModel(iterations, new TwoPassDataIndexer(es, cut));
+ }
+
public static void mergeReportIntoManifest(Map<String, String> manifest,
Map<String, String> report, String namespace) {
@@ -309,4 +318,27 @@ public class Parser extends AbstractBottomUpParser {
posModel, chunkModel, (opennlp.tools.parser.HeadRules) rules,
ParserType.CHUNKING, manifestInfoEntries);
}
+
+ /**
+ * @deprecated use {@link #train(String, ObjectStream, HeadRules, TrainingParameters)}
+ * instead and pass in a TrainingParameters object.
+ */
+ @Deprecated
+ public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut)
+ throws IOException {
+
+ TrainingParameters params = new TrainingParameters();
+ params.put("dict", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
+
+ params.put("tagger", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
+ params.put("tagger", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
+ params.put("chunker", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
+ params.put("chunker", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
+ params.put("check", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
+ params.put("check", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
+ params.put("build", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
+ params.put("build", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
+
+ return train(languageCode, parseSamples, rules, params);
+ }
}
[32/50] [abbrv] opennlp git commit: Remove deprecated API from the
Parser
Posted by jo...@apache.org.
Remove deprecated API from the Parser
See issue OPENNLP-886
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/5f36766c
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/5f36766c
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/5f36766c
Branch: refs/heads/889
Commit: 5f36766c1707bdfff7ebff550b172f937787b2d4
Parents: 4f27908
Author: Joern Kottmann <jo...@apache.org>
Authored: Mon Dec 19 19:49:31 2016 +0100
Committer: Joern Kottmann <jo...@apache.org>
Committed: Tue Dec 20 17:51:09 2016 +0100
----------------------------------------------------------------------
.../cmdline/parser/BuildModelUpdaterTool.java | 3 +--
.../opennlp/tools/parser/chunking/Parser.java | 23 --------------------
.../tools/parser/chunking/ParserTest.java | 3 ++-
3 files changed, 3 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/5f36766c/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/BuildModelUpdaterTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/BuildModelUpdaterTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/BuildModelUpdaterTool.java
index 3d8695b..fe98725 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/BuildModelUpdaterTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/BuildModelUpdaterTool.java
@@ -49,8 +49,7 @@ public final class BuildModelUpdaterTool extends ModelUpdaterTool {
System.out.println("Training builder");
ObjectStream<Event> bes = new ParserEventStream(parseSamples,
originalModel.getHeadRules(), ParserEventTypeEnum.BUILD, mdict);
- AbstractModel buildModel = Parser.train(bes,
- 100, 5);
+ AbstractModel buildModel = Parser.train(bes, 100, 5);
parseSamples.close();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/5f36766c/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
index ae20e8a..505f688 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
@@ -318,27 +318,4 @@ public class Parser extends AbstractBottomUpParser {
posModel, chunkModel, rules,
ParserType.CHUNKING, manifestInfoEntries);
}
-
- /**
- * @deprecated use {@link #train(String, ObjectStream, HeadRules, TrainingParameters)}
- * instead and pass in a TrainingParameters object.
- */
- @Deprecated
- public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut)
- throws IOException {
-
- TrainingParameters params = new TrainingParameters();
- params.put("dict", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
-
- params.put("tagger", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("tagger", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put("chunker", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("chunker", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put("check", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("check", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
- params.put("build", TrainingParameters.CUTOFF_PARAM, Integer.toString(cut));
- params.put("build", TrainingParameters.ITERATIONS_PARAM, Integer.toString(iterations));
-
- return train(languageCode, parseSamples, rules, params);
- }
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/5f36766c/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
index 60f80a3..e9c6006 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/parser/chunking/ParserTest.java
@@ -27,6 +27,7 @@ import opennlp.tools.parser.ParserModel;
import opennlp.tools.parser.ParserTestUtil;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.TrainingParameters;
import org.junit.Test;
/**
@@ -44,7 +45,7 @@ public class ParserTest {
ObjectStream<Parse> parseSamples = ParserTestUtil.openTestTrainingData();
HeadRules headRules = ParserTestUtil.createTestHeadRules();
- ParserModel model = Parser.train("en", parseSamples, headRules, 100, 0);
+ ParserModel model = Parser.train("en", parseSamples, headRules, TrainingParameters.defaultParams());
opennlp.tools.parser.Parser parser = ParserFactory.create(model);
[27/50] [abbrv] opennlp git commit: Speed up GIS training by saving
Executor in the GISTrainer
Posted by jo...@apache.org.
Speed up GIS training by saving Executor in the GISTrainer
Thanks to Daniel Russ for providing a patch
See issue OPENNLP-759
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/664d8033
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/664d8033
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/664d8033
Branch: refs/heads/889
Commit: 664d80330a3b4df05a524c3c202fe3f4de2806ba
Parents: 0637180
Author: Joern Kottmann <ko...@gmail.com>
Authored: Mon Dec 19 13:01:43 2016 +0100
Committer: Joern Kottmann <jo...@apache.org>
Committed: Tue Dec 20 09:40:28 2016 +0100
----------------------------------------------------------------------
.../opennlp/tools/ml/maxent/GISTrainer.java | 34 +++++++++++---------
1 file changed, 18 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/664d8033/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
index 0527979..9919bb0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
@@ -23,7 +23,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -403,6 +405,9 @@ class GISTrainer {
/* Estimate and return the model parameters. */
private void findParameters(int iterations, double correctionConstant) {
+ int threads=modelExpects.length;
+ ExecutorService executor = Executors.newFixedThreadPool(threads);
+ CompletionService<ModelExpactationComputeTask> completionService=new ExecutorCompletionService<GISTrainer.ModelExpactationComputeTask>(executor);
double prevLL = 0.0;
double currLL;
display("Performing " + iterations + " iterations.\n");
@@ -413,7 +418,7 @@ class GISTrainer {
display(" " + i + ": ");
else
display(i + ": ");
- currLL = nextIteration(correctionConstant);
+ currLL = nextIteration(correctionConstant,completionService);
if (i > 1) {
if (prevLL > currLL) {
System.err.println("Model Diverging: loglikelihood decreased");
@@ -431,6 +436,7 @@ class GISTrainer {
modelExpects = null;
numTimesEventsSeen = null;
contexts = null;
+ executor.shutdown();
}
//modeled on implementation in Zhang Le's maxent kit
@@ -544,34 +550,32 @@ class GISTrainer {
}
/* Compute one iteration of GIS and retutn log-likelihood.*/
- private double nextIteration(double correctionConstant) {
+ private double nextIteration(double correctionConstant, CompletionService<ModelExpactationComputeTask> completionService) {
// compute contribution of p(a|b_i) for each feature and the new
// correction parameter
double loglikelihood = 0.0;
int numEvents = 0;
int numCorrect = 0;
+ // Each thread gets equal number of tasks, if the number of tasks
+ // is not divisible by the number of threads, the first "leftOver"
+ // threads have one extra task.
int numberOfThreads = modelExpects.length;
-
- ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
-
int taskSize = numUniqueEvents / numberOfThreads;
-
int leftOver = numUniqueEvents % numberOfThreads;
- List<Future<?>> futures = new ArrayList<Future<?>>();
-
+ // submit all tasks to the completion service.
for (int i = 0; i < numberOfThreads; i++) {
- if (i != numberOfThreads - 1)
- futures.add(executor.submit(new ModelExpactationComputeTask(i, i*taskSize, taskSize)));
+ if (i < leftOver)
+ completionService.submit(new ModelExpactationComputeTask(i, i*taskSize+i, taskSize+1));
else
- futures.add(executor.submit(new ModelExpactationComputeTask(i, i*taskSize, taskSize + leftOver)));
+ completionService.submit(new ModelExpactationComputeTask(i, i*taskSize+leftOver, taskSize));
}
- for (Future<?> future : futures) {
- ModelExpactationComputeTask finishedTask;
+ for (int i=0; i<numberOfThreads; i++) {
+ ModelExpactationComputeTask finishedTask = null;
try {
- finishedTask = (ModelExpactationComputeTask) future.get();
+ finishedTask = completionService.take().get();
} catch (InterruptedException e) {
// TODO: We got interrupted, but that is currently not really supported!
// For now we just print the exception and fail hard. We hopefully soon
@@ -591,8 +595,6 @@ class GISTrainer {
loglikelihood += finishedTask.getLoglikelihood();
}
- executor.shutdown();
-
display(".");
// merge the results of the two computations
[44/50] [abbrv] opennlp git commit: OPENNLP-871: Cleanup for Java 8
and remove deprecated tokenizer code
Posted by jo...@apache.org.
OPENNLP-871: Cleanup for Java 8 and remove deprecated tokenizer code
Closes #17
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c49a87ab
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c49a87ab
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c49a87ab
Branch: refs/heads/889
Commit: c49a87abdb6c07a123a2234f4ec5ca3d21306d69
Parents: 99323ad
Author: smarthi <sm...@apache.org>
Authored: Thu Dec 22 22:39:33 2016 -0500
Committer: Kottmann <jo...@apache.org>
Committed: Fri Dec 23 16:46:42 2016 +0100
----------------------------------------------------------------------
.../tools/cmdline/chunker/ChunkerMETool.java | 6 +-
.../opennlp/tools/dictionary/Dictionary.java | 7 +-
.../tools/tokenize/TokenizerCrossValidator.java | 36 --------
.../tools/tokenize/TokenizerFactory.java | 22 ++---
.../opennlp/tools/tokenize/TokenizerME.java | 89 --------------------
.../opennlp/tools/tokenize/TokenizerModel.java | 53 ------------
.../opennlp/tools/eval/Conll00ChunkerEval.java | 3 +-
7 files changed, 13 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
index 5b87c9e..b511a0b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerMETool.java
@@ -69,11 +69,9 @@ public class ChunkerMETool extends BasicCmdLineTool {
continue;
}
- String[] chunks = chunker.chunk(posSample.getSentence(),
- posSample.getTags());
+ String[] chunks = chunker.chunk(posSample.getSentence(), posSample.getTags());
- System.out.println(new ChunkSample(posSample.getSentence(),
- posSample.getTags(), chunks).nicePrint());
+ System.out.println(new ChunkSample(posSample.getSentence(), posSample.getTags(), chunks).nicePrint());
perfMon.incrementCounter();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
index 58b7a6e..4961741 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/dictionary/Dictionary.java
@@ -31,7 +31,6 @@ import java.util.StringTokenizer;
import opennlp.tools.dictionary.serializer.Attributes;
import opennlp.tools.dictionary.serializer.DictionarySerializer;
import opennlp.tools.dictionary.serializer.Entry;
-import opennlp.tools.dictionary.serializer.EntryInserter;
import opennlp.tools.util.StringList;
import opennlp.tools.util.StringUtil;
@@ -113,11 +112,7 @@ public class Dictionary implements Iterable<StringList> {
* @throws IOException
*/
public Dictionary(InputStream in) throws IOException {
- isCaseSensitive = DictionarySerializer.create(in, new EntryInserter() {
- public void insert(Entry entry) {
- put(entry.getTokens());
- }
- });
+ isCaseSensitive = DictionarySerializer.create(in, entry -> put(entry.getTokens()));
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
index 3ca3c1d..fe9e4c6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
@@ -18,13 +18,10 @@
package opennlp.tools.tokenize;
import java.io.IOException;
-
-import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
import opennlp.tools.util.eval.FMeasure;
-import opennlp.tools.util.model.ModelUtil;
public class TokenizerCrossValidator {
@@ -42,39 +39,6 @@ public class TokenizerCrossValidator {
}
/**
- * @deprecated use
- * {@link #TokenizerCrossValidator(TrainingParameters, TokenizerFactory, TokenizerEvaluationMonitor...)}
- * instead and pass in a {@link TokenizerFactory}
- */
- public TokenizerCrossValidator(String language, Dictionary abbreviations,
- boolean alphaNumericOptimization, TrainingParameters params,
- TokenizerEvaluationMonitor ... listeners) {
- this(params, new TokenizerFactory(language, abbreviations,
- alphaNumericOptimization, null), listeners);
- }
-
- /**
- * @deprecated use
- * {@link #TokenizerCrossValidator(TrainingParameters, TokenizerFactory, TokenizerEvaluationMonitor...)}
- * instead and pass in a {@link TokenizerFactory}
- */
- public TokenizerCrossValidator(String language, boolean alphaNumericOptimization) {
- this(language, alphaNumericOptimization, ModelUtil.createDefaultTrainingParameters());
- }
-
- /**
- * @deprecated use
- * {@link #TokenizerCrossValidator(TrainingParameters, TokenizerFactory, TokenizerEvaluationMonitor...)}
- * instead and pass in a {@link TokenizerFactory}
- */
- public TokenizerCrossValidator(String language,
- boolean alphaNumericOptimization, TrainingParameters params,
- TokenizerEvaluationMonitor ... listeners) {
- this(language, null, alphaNumericOptimization, params, listeners);
- }
-
-
- /**
* Starts the evaluation.
*
* @param samples
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
index f9e789a..ffa793b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerFactory.java
@@ -37,7 +37,7 @@ public class TokenizerFactory extends BaseToolFactory {
private String languageCode;
private Dictionary abbreviationDictionary;
- private Boolean useAlphaNumericOptimization;
+ private Boolean useAlphaNumericOptimization = false;
private Pattern alphaNumericPattern;
private static final String ABBREVIATIONS_ENTRY_NAME = "abbreviations.dictionary";
@@ -112,13 +112,11 @@ public class TokenizerFactory extends BaseToolFactory {
public Map<String, String> createManifestEntries() {
Map<String, String> manifestEntries = super.createManifestEntries();
- manifestEntries.put(USE_ALPHA_NUMERIC_OPTIMIZATION,
- Boolean.toString(isUseAlphaNumericOptmization()));
+ manifestEntries.put(USE_ALPHA_NUMERIC_OPTIMIZATION, Boolean.toString(isUseAlphaNumericOptmization()));
// alphanumeric pattern is optional
if (getAlphaNumericPattern() != null)
- manifestEntries.put(ALPHA_NUMERIC_PATTERN, getAlphaNumericPattern()
- .pattern());
+ manifestEntries.put(ALPHA_NUMERIC_PATTERN, getAlphaNumericPattern().pattern());
return manifestEntries;
}
@@ -167,9 +165,8 @@ public class TokenizerFactory extends BaseToolFactory {
*/
public Pattern getAlphaNumericPattern() {
if (this.alphaNumericPattern == null) {
- if (artifactProvider != null) {
- String prop = this.artifactProvider
- .getManifestProperty(ALPHA_NUMERIC_PATTERN);
+ if (this.artifactProvider != null) {
+ String prop = this.artifactProvider.getManifestProperty(ALPHA_NUMERIC_PATTERN);
if (prop != null) {
this.alphaNumericPattern = Pattern.compile(prop);
}
@@ -189,8 +186,8 @@ public class TokenizerFactory extends BaseToolFactory {
* @return true if the alpha numeric optimization is enabled, otherwise false
*/
public boolean isUseAlphaNumericOptmization() {
- if (this.useAlphaNumericOptimization == null && artifactProvider != null) {
- this.useAlphaNumericOptimization = Boolean.valueOf(artifactProvider
+ if (artifactProvider != null) {
+ this.useAlphaNumericOptimization = Boolean.valueOf(this.artifactProvider
.getManifestProperty(USE_ALPHA_NUMERIC_OPTIMIZATION));
}
return this.useAlphaNumericOptimization;
@@ -203,8 +200,7 @@ public class TokenizerFactory extends BaseToolFactory {
*/
public Dictionary getAbbreviationDictionary() {
if (this.abbreviationDictionary == null && artifactProvider != null) {
- this.abbreviationDictionary = artifactProvider
- .getArtifact(ABBREVIATIONS_ENTRY_NAME);
+ this.abbreviationDictionary = this.artifactProvider.getArtifact(ABBREVIATIONS_ENTRY_NAME);
}
return this.abbreviationDictionary;
}
@@ -215,7 +211,7 @@ public class TokenizerFactory extends BaseToolFactory {
* @return the language code
*/
public String getLanguageCode() {
- if (this.languageCode == null && artifactProvider != null) {
+ if (this.languageCode == null && this.artifactProvider != null) {
this.languageCode = this.artifactProvider.getLanguage();
}
return this.languageCode;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
index 4c4c638..491b6fa 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerME.java
@@ -34,7 +34,6 @@ import opennlp.tools.tokenize.lang.Factory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.TrainingParameters;
-import opennlp.tools.util.model.ModelUtil;
/**
* A Tokenizer for converting raw text into separated tokens. It uses
@@ -254,94 +253,6 @@ public class TokenizerME extends AbstractTokenizer {
}
/**
- * Trains a model for the {@link TokenizerME}.
- *
- * @param languageCode the language of the natural text
- * @param samples the samples used for the training.
- * @param useAlphaNumericOptimization - if true alpha numerics are skipped
- * @param mlParams the machine learning train parameters
- *
- * @return the trained {@link TokenizerModel}
- *
- * @throws IOException it throws an {@link IOException} if an {@link IOException}
- * is thrown during IO operations on a temp file which is created during training.
- * Or if reading from the {@link ObjectStream} fails.
- *
- * @deprecated Use
- * {@link #train(ObjectStream, TokenizerFactory, TrainingParameters)}
- * and pass in a {@link TokenizerFactory}
- */
- public static TokenizerModel train(String languageCode, ObjectStream<TokenSample> samples,
- boolean useAlphaNumericOptimization, TrainingParameters mlParams) throws IOException {
- return train(languageCode, samples, null, useAlphaNumericOptimization,
- mlParams);
- }
-
- /**
- * Trains a model for the {@link TokenizerME}.
- *
- * @param languageCode the language of the natural text
- * @param samples the samples used for the training.
- * @param abbreviations an abbreviations dictionary
- * @param useAlphaNumericOptimization - if true alpha numerics are skipped
- * @param mlParams the machine learning train parameters
- *
- * @return the trained {@link TokenizerModel}
- *
- * @throws IOException it throws an {@link IOException} if an {@link IOException}
- * is thrown during IO operations on a temp file which is created during training.
- * Or if reading from the {@link ObjectStream} fails.
- *
- * @deprecated Use
- * {@link #train(ObjectStream, TokenizerFactory, TrainingParameters)}
- * and pass in a {@link TokenizerFactory}
- */
- public static TokenizerModel train(String languageCode,
- ObjectStream<TokenSample> samples, Dictionary abbreviations,
- boolean useAlphaNumericOptimization, TrainingParameters mlParams)
- throws IOException {
- Factory factory = new Factory();
-
- Map<String, String> manifestInfoEntries = new HashMap<>();
-
- ObjectStream<Event> eventStream = new TokSpanEventStream(samples,
- useAlphaNumericOptimization, factory.getAlphanumeric(languageCode),
- factory.createTokenContextGenerator(languageCode,
- getAbbreviations(abbreviations)));
-
- EventTrainer trainer = TrainerFactory.getEventTrainer(
- mlParams.getSettings(), manifestInfoEntries);
-
- MaxentModel maxentModel = trainer.train(eventStream);
-
- return new TokenizerModel(languageCode, maxentModel, abbreviations,
- useAlphaNumericOptimization, manifestInfoEntries);
- }
-
-
- /**
- * Trains a model for the {@link TokenizerME} with a default cutoff of 5 and 100 iterations.
- *
- * @param languageCode the language of the natural text
- * @param samples the samples used for the training.
- * @param useAlphaNumericOptimization - if true alpha numerics are skipped
- *
- * @return the trained {@link TokenizerModel}
- *
- * @throws IOException it throws an {@link IOException} if an {@link IOException}
- * is thrown during IO operations on a temp file which is
- *
- * @deprecated Use
- * {@link #train(ObjectStream, TokenizerFactory, TrainingParameters)}
- * and pass in a {@link TokenizerFactory}
- */
- public static TokenizerModel train(String languageCode, ObjectStream<TokenSample> samples,
- boolean useAlphaNumericOptimization) throws IOException {
- return train(samples, TokenizerFactory.create(null, languageCode, null, useAlphaNumericOptimization, null),
- ModelUtil.createDefaultTrainingParameters());
- }
-
- /**
* Returns the value of the alpha-numeric optimization flag.
*
* @return true if the tokenizer should use alpha-numeric optimization, false otherwise.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
index e63b946..ed84b4e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerModel.java
@@ -66,59 +66,6 @@ public final class TokenizerModel extends BaseModel {
/**
* Initializes the current instance.
*
- * @param language the language the tokenizer should use
- * @param tokenizerMaxentModel the statistical model of the tokenizer
- * @param abbreviations the dictionary containing the abbreviations
- * @param useAlphaNumericOptimization if true alpha numeric optimization is enabled, otherwise not
- * @param manifestInfoEntries the additional meta data which should be written into manifest
- *
- * @deprecated Use
- * {@link TokenizerModel#TokenizerModel(MaxentModel, Map, TokenizerFactory)}
- * instead and pass in a {@link TokenizerFactory}.
- */
- public TokenizerModel(String language, MaxentModel tokenizerMaxentModel,
- Dictionary abbreviations, boolean useAlphaNumericOptimization,
- Map<String, String> manifestInfoEntries) {
- this(tokenizerMaxentModel, manifestInfoEntries,
- new TokenizerFactory(language, abbreviations, useAlphaNumericOptimization, null));
- }
-
- /**
- * Initializes the current instance.
- *
- * @param language the language the tokenizer should use
- * @param tokenizerMaxentModel the statistical model of the tokenizer
- * @param useAlphaNumericOptimization if true alpha numeric optimization is enabled, otherwise not
- * @param manifestInfoEntries the additional meta data which should be written into manifest
- *
- * @deprecated Use
- * {@link TokenizerModel#TokenizerModel(MaxentModel, Map, TokenizerFactory)}
- * instead and pass in a {@link TokenizerFactory}.
- */
- public TokenizerModel(String language, AbstractModel tokenizerMaxentModel,
- boolean useAlphaNumericOptimization, Map<String, String> manifestInfoEntries) {
- this(language, tokenizerMaxentModel, null, useAlphaNumericOptimization, manifestInfoEntries);
- }
-
- /**
- * Initializes the current instance.
- *
- * @param language the language the tokenizer should use
- * @param tokenizerMaxentModel the statistical model of the tokenizer
- * @param useAlphaNumericOptimization if true alpha numeric optimization is enabled, otherwise not
- *
- * @deprecated Use
- * {@link TokenizerModel#TokenizerModel(MaxentModel, Map, TokenizerFactory)}
- * instead and pass in a {@link TokenizerFactory}.
- */
- public TokenizerModel(String language, AbstractModel tokenizerMaxentModel,
- boolean useAlphaNumericOptimization) {
- this(language, tokenizerMaxentModel, useAlphaNumericOptimization, null);
- }
-
- /**
- * Initializes the current instance.
- *
* @param in the Input Stream to load the model from
*
* @throws IOException if reading from the stream fails in anyway
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c49a87ab/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
index 3b8e060..0ff3002 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/Conll00ChunkerEval.java
@@ -59,8 +59,7 @@ public class Conll00ChunkerEval {
double expectedFMeasure) throws IOException {
ObjectStream<ChunkSample> samples = new ChunkSampleStream(
- new PlainTextByLineStream(new MarkableFileInputStreamFactory(testData),
- "UTF-8"));
+ new PlainTextByLineStream(new MarkableFileInputStreamFactory(testData),"UTF-8"));
ChunkerEvaluator evaluator = new ChunkerEvaluator(new ChunkerME(model));
evaluator.evaluate(samples);
[24/50] [abbrv] opennlp git commit: Collapse duplicate catch blocks
into multi-catch
Posted by jo...@apache.org.
Collapse duplicate catch blocks into multi-catch
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/1550dac1
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/1550dac1
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/1550dac1
Branch: refs/heads/889
Commit: 1550dac129b2db957db09de0456e3c579119a741
Parents: 6871d11
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 20:18:18 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../formats/frenchtreebank/ConstitParseSampleStream.java | 4 +---
.../main/java/opennlp/tools/util/ext/ExtensionLoader.java | 8 ++------
2 files changed, 3 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1550dac1/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
index 00dfbfb..e908cb1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
@@ -44,9 +44,7 @@ public class ConstitParseSampleStream extends FilterObjectStream<byte[], Parse>
SAXParserFactory factory = SAXParserFactory.newInstance();
try {
saxParser = factory.newSAXParser();
- } catch (ParserConfigurationException e) {
- throw new IllegalStateException(e);
- } catch (SAXException e) {
+ } catch (ParserConfigurationException | SAXException e) {
throw new IllegalStateException(e);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1550dac1/opennlp-tools/src/main/java/opennlp/tools/util/ext/ExtensionLoader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/ext/ExtensionLoader.java b/opennlp-tools/src/main/java/opennlp/tools/util/ext/ExtensionLoader.java
index 93a520b..b33d577 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/ext/ExtensionLoader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/ext/ExtensionLoader.java
@@ -75,17 +75,13 @@ public class ExtensionLoader {
Field instanceField;
try {
instanceField = extClazz.getDeclaredField("INSTANCE");
- } catch (NoSuchFieldException e1) {
- throw new ExtensionNotLoadedException(e1);
- } catch (SecurityException e1) {
+ } catch (NoSuchFieldException | SecurityException e1) {
throw new ExtensionNotLoadedException(e1);
}
if(instanceField != null) {
try {
return (T) instanceField.get(null);
- } catch (IllegalArgumentException e1) {
- throw new ExtensionNotLoadedException(e1);
- } catch (IllegalAccessException e1) {
+ } catch (IllegalArgumentException | IllegalAccessException e1) {
throw new ExtensionNotLoadedException(e1);
}
}
[45/50] [abbrv] opennlp git commit: Update Jersey to 2.25 and use
Grizzly HTTP Server
Posted by jo...@apache.org.
Update Jersey to 2.25 and use Grizzly HTTP Server
See issue OPENNNLP-897
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/1d1c4194
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/1d1c4194
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/1d1c4194
Branch: refs/heads/889
Commit: 1d1c41945a50a740519131bfff4e1875de69dd6b
Parents: c49a87a
Author: Joern Kottmann <jo...@apache.org>
Authored: Wed Dec 21 14:55:24 2016 +0100
Committer: Kottmann <jo...@apache.org>
Committed: Fri Dec 23 19:47:24 2016 +0100
----------------------------------------------------------------------
opennlp-brat-annotator/pom.xml | 24 +++---------
.../opennlp/bratann/NameFinderAnnService.java | 41 ++++++--------------
2 files changed, 18 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1d1c4194/opennlp-brat-annotator/pom.xml
----------------------------------------------------------------------
diff --git a/opennlp-brat-annotator/pom.xml b/opennlp-brat-annotator/pom.xml
index 65e3b69..1007f61 100644
--- a/opennlp-brat-annotator/pom.xml
+++ b/opennlp-brat-annotator/pom.xml
@@ -33,27 +33,15 @@
<dependencies>
<dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- <version>9.2.3.v20140905</version>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-grizzly2-http</artifactId>
+ <version>2.25</version>
</dependency>
<dependency>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- <version>9.2.3.v20140905</version>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-bundle</artifactId>
- <version>1.18.1</version>
- </dependency>
-
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- <version>1.18.1</version>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <version>2.25</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/opennlp/blob/1d1c4194/opennlp-brat-annotator/src/main/java/opennlp/bratann/NameFinderAnnService.java
----------------------------------------------------------------------
diff --git a/opennlp-brat-annotator/src/main/java/opennlp/bratann/NameFinderAnnService.java b/opennlp-brat-annotator/src/main/java/opennlp/bratann/NameFinderAnnService.java
index 60d2a1b..8e2aef8 100644
--- a/opennlp-brat-annotator/src/main/java/opennlp/bratann/NameFinderAnnService.java
+++ b/opennlp-brat-annotator/src/main/java/opennlp/bratann/NameFinderAnnService.java
@@ -18,13 +18,11 @@
package opennlp.bratann;
import java.io.File;
+import java.net.URI;
import java.util.Arrays;
import java.util.List;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-
+import com.sun.net.httpserver.HttpServer;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinder;
import opennlp.tools.namefind.TokenNameFinderModel;
@@ -37,6 +35,10 @@ import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.tokenize.WhitespaceTokenizer;
+import org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpServerFactory;
+import org.glassfish.jersey.server.ResourceConfig;
+
+import javax.ws.rs.core.UriBuilder;
public class NameFinderAnnService {
@@ -47,9 +49,10 @@ public class NameFinderAnnService {
public static void main(String[] args) throws Exception {
if (args.length == 0) {
+ System.out.println("Usage:");
System.out.println(
- "[-serverPort port] [-tokenizerModel file] [-ruleBasedTokenizer whitespace|simple] [-sentenceDetectorModel file] "
- + "namefinderFile|nameFinderURI");
+ "[NameFinderAnnService -serverPort port] [-tokenizerModel file] [-ruleBasedTokenizer whitespace|simple] " +
+ "[-sentenceDetectorModel file] namefinderFile|nameFinderURI");
return;
}
@@ -92,28 +95,8 @@ public class NameFinderAnnService {
nameFinders = new TokenNameFinder[] { new NameFinderME(
new TokenNameFinderModel(new File(args[args.length - 1]))) };
- ServletContextHandler context = new ServletContextHandler(
- ServletContextHandler.SESSIONS);
- context.setContextPath("/");
-
- Server jettyServer = new Server(serverPort);
- jettyServer.setHandler(context);
-
- ServletHolder jerseyServlet = context
- .addServlet(com.sun.jersey.spi.container.servlet.ServletContainer.class, "/*");
- jerseyServlet.setInitParameter("com.sun.jersey.config.property.packages",
- "opennlp.bratann");
- jerseyServlet.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature", "true");
- jerseyServlet.setInitOrder(0);
-
- jerseyServlet.setInitParameter("jersey.config.server.provider.classnames",
- NameFinderResource.class.getCanonicalName());
-
- try {
- jettyServer.start();
- jettyServer.join();
- } finally {
- jettyServer.destroy();
- }
+ URI baseUri = UriBuilder.fromUri("http://localhost/").port(serverPort).build();
+ ResourceConfig config = new ResourceConfig(NameFinderResource.class);
+ GrizzlyHttpServerFactory.createHttpServer(baseUri, config);
}
}
[46/50] [abbrv] opennlp git commit: OPENNLP-871: Clean up code base
for release
Posted by jo...@apache.org.
OPENNLP-871: Clean up code base for release
This closes #20
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/9cb610bc
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/9cb610bc
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/9cb610bc
Branch: refs/heads/889
Commit: 9cb610bcbc3c541a3326d6504056d5526876c4cb
Parents: 1d1c419
Author: smarthi <sm...@apache.org>
Authored: Fri Dec 23 20:52:40 2016 -0500
Committer: Kottmann <jo...@apache.org>
Committed: Sat Dec 24 16:27:49 2016 +0100
----------------------------------------------------------------------
.../brat/BratNameSampleStreamFactory.java | 5 +-
.../ConstitParseSampleStream.java | 6 +-
.../formats/muc/DocumentSplitterStream.java | 2 +-
.../muc/Muc6NameSampleStreamFactory.java | 10 +--
.../tools/formats/muc/MucElementNames.java | 2 +-
.../formats/muc/MucNameContentHandler.java | 12 +--
.../tools/formats/muc/MucNameSampleStream.java | 2 +-
.../opennlp/tools/formats/muc/SgmlParser.java | 2 +-
.../formats/ontonotes/DocumentToLineStream.java | 2 +-
.../ontonotes/OntoNotesNameSampleStream.java | 8 +-
.../OntoNotesNameSampleStreamFactory.java | 20 ++---
.../OntoNotesParseSampleStreamFactory.java | 18 ++---
.../tools/lemmatizer/DictionaryLemmatizer.java | 5 +-
.../opennlp/tools/ml/maxent/GISTrainer.java | 14 ++--
.../tools/ml/model/GenericModelReader.java | 25 ++++---
.../ml/model/OnePassRealValueDataIndexer.java | 6 +-
.../naivebayes/BinaryNaiveBayesModelWriter.java | 2 +-
.../tools/ml/naivebayes/NaiveBayesTrainer.java | 2 +-
.../perceptron/BinaryPerceptronModelWriter.java | 2 +-
.../ml/perceptron/PerceptronModelWriter.java | 8 +-
.../tools/ml/perceptron/PerceptronTrainer.java | 43 ++++-------
.../PlainTextPerceptronModelWriter.java | 6 +-
.../SimplePerceptronSequenceTrainer.java | 47 ++++++------
.../opennlp/tools/namefind/NameFinderME.java | 14 ++--
.../java/opennlp/tools/ngram/NGramModel.java | 11 +--
.../tools/parser/AbstractBottomUpParser.java | 6 +-
.../tools/parser/AbstractContextGenerator.java | 1 -
.../tools/parser/ChunkContextGenerator.java | 2 +-
.../treeinsert/AttachContextGenerator.java | 10 +--
.../treeinsert/BuildContextGenerator.java | 2 +-
.../treeinsert/CheckContextGenerator.java | 2 +-
.../opennlp/tools/parser/treeinsert/Parser.java | 34 +++------
.../parser/treeinsert/ParserEventStream.java | 77 ++++++++++----------
.../java/opennlp/tools/postag/POSModel.java | 8 +-
.../java/opennlp/tools/postag/POSTaggerME.java | 15 ++--
.../opennlp/tools/util/BaseToolFactory.java | 6 +-
.../tools/eval/OntoNotes4NameFinderEval.java | 22 ++----
.../tools/eval/OntoNotes4ParserEval.java | 22 ++----
.../tools/eval/OntoNotes4PosTaggerEval.java | 22 ++----
.../tools/postag/POSTaggerFactoryTest.java | 28 ++++---
40 files changed, 219 insertions(+), 312 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
index b7a8c41..669c699 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/brat/BratNameSampleStreamFactory.java
@@ -18,10 +18,7 @@
package opennlp.tools.formats.brat;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
-import java.io.InputStream;
-
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
@@ -75,7 +72,7 @@ public class BratNameSampleStreamFactory extends AbstractSampleStreamFactory<Nam
* Checks that non of the passed values are null.
*
* @param objects
- * @return
+ * @return true or false
*/
private boolean notNull(Object... objects) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
index e908cb1..1964131 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitParseSampleStream.java
@@ -36,7 +36,7 @@ public class ConstitParseSampleStream extends FilterObjectStream<byte[], Parse>
private SAXParser saxParser;
- private List<Parse> parses = new ArrayList<Parse>();
+ private List<Parse> parses = new ArrayList<>();
protected ConstitParseSampleStream(ObjectStream<byte[]> samples) {
super(samples);
@@ -57,12 +57,12 @@ public class ConstitParseSampleStream extends FilterObjectStream<byte[], Parse>
if (xmlbytes != null) {
- List<Parse> producedParses = new ArrayList<Parse>();
+ List<Parse> producedParses = new ArrayList<>();
try {
saxParser.parse(new ByteArrayInputStream(xmlbytes), new ConstitDocumentHandler(producedParses));
} catch (SAXException e) {
//TODO update after Java6 upgrade
- throw (IOException) new IOException(e.getMessage()).initCause(e);
+ throw new IOException(e.getMessage(), e);
}
parses.addAll(producedParses);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/muc/DocumentSplitterStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/DocumentSplitterStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/DocumentSplitterStream.java
index 257505d..9ac9514 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/DocumentSplitterStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/DocumentSplitterStream.java
@@ -30,7 +30,7 @@ class DocumentSplitterStream extends FilterObjectStream<String, String> {
private static final String DOC_START_ELEMENT = "<DOC>";
private static final String DOC_END_ELEMENT = "</DOC>";
- private List<String> docs = new ArrayList<String>();
+ private List<String> docs = new ArrayList<>();
DocumentSplitterStream(ObjectStream<String> samples) {
super(samples);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
index b76613c..496253b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/Muc6NameSampleStreamFactory.java
@@ -18,9 +18,7 @@
package opennlp.tools.formats.muc;
import java.io.File;
-import java.io.FileFilter;
import java.nio.charset.Charset;
-
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.StreamFactoryRegistry;
@@ -55,12 +53,8 @@ public class Muc6NameSampleStreamFactory extends AbstractSampleStreamFactory<Nam
Tokenizer tokenizer = new TokenizerME(tokenizerModel);
ObjectStream<String> mucDocStream = new FileToStringSampleStream(
- new DirectorySampleStream(params.getData(), new FileFilter() {
-
- public boolean accept(File file) {
- return StringUtil.toLowerCase(file.getName()).endsWith(".sgm");
- }
- }, false), Charset.forName("UTF-8"));
+ new DirectorySampleStream(params.getData(),
+ file -> StringUtil.toLowerCase(file.getName()).endsWith(".sgm"), false), Charset.forName("UTF-8"));
return new MucNameSampleStream(tokenizer, mucDocStream);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
index 35b499d..6574fd0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucElementNames.java
@@ -32,7 +32,7 @@ class MucElementNames {
static final Set<String> CONTENT_ELEMENTS;
static {
- Set<String> contentElementNames = new HashSet<String>();
+ Set<String> contentElementNames = new HashSet<>();
contentElementNames.add(MucElementNames.HEADLINE_ELEMENT);
contentElementNames.add(MucElementNames.DATELINE_ELEMENT);
contentElementNames.add(MucElementNames.DD_ELEMENT);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameContentHandler.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameContentHandler.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameContentHandler.java
index 7300251..4d6d3a4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameContentHandler.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameContentHandler.java
@@ -43,7 +43,7 @@ public class MucNameContentHandler extends SgmlParser.ContentHandler {
private static final Set<String> EXPECTED_TYPES;
static {
- Set<String> types = new HashSet<String>();
+ Set<String> types = new HashSet<>();
types.add("PERSON");
types.add("ORGANIZATION");
@@ -55,7 +55,7 @@ public class MucNameContentHandler extends SgmlParser.ContentHandler {
EXPECTED_TYPES = Collections.unmodifiableSet(types);
- Set<String> nameElements = new HashSet<String>();
+ Set<String> nameElements = new HashSet<>();
nameElements.add(ENTITY_ELEMENT_NAME);
nameElements.add(TIME_ELEMENT_NAME);
nameElements.add(NUM_ELEMENT_NAME);
@@ -65,12 +65,12 @@ public class MucNameContentHandler extends SgmlParser.ContentHandler {
private final Tokenizer tokenizer;
private final List<NameSample> storedSamples;
- boolean isInsideContentElement = false;
- private final List<String> text = new ArrayList<String>();
+ private boolean isInsideContentElement = false;
+ private final List<String> text = new ArrayList<>();
private boolean isClearAdaptiveData = false;
- private final Stack<Span> incompleteNames = new Stack<Span>();
+ private final Stack<Span> incompleteNames = new Stack<>();
- private List<Span> names = new ArrayList<Span>();
+ private List<Span> names = new ArrayList<>();
public MucNameContentHandler(Tokenizer tokenizer,
List<NameSample> storedSamples) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
index 530302d..281df5d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/MucNameSampleStream.java
@@ -31,7 +31,7 @@ public class MucNameSampleStream extends FilterObjectStream<String, NameSample>
private final Tokenizer tokenizer;
- private List<NameSample> storedSamples = new ArrayList<NameSample>();
+ private List<NameSample> storedSamples = new ArrayList<>();
protected MucNameSampleStream(Tokenizer tokenizer, ObjectStream<String> samples) {
super(samples);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/muc/SgmlParser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/SgmlParser.java b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/SgmlParser.java
index fd18f6f..e85e995 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/muc/SgmlParser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/muc/SgmlParser.java
@@ -75,7 +75,7 @@ public class SgmlParser {
// value chars
// " <- end
- Map<String, String> attributes = new HashMap<String, String>();
+ Map<String, String> attributes = new HashMap<>();
StringBuilder key = new StringBuilder();
StringBuilder value = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/DocumentToLineStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/DocumentToLineStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/DocumentToLineStream.java
index 4cab6ea..eb8b3e2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/DocumentToLineStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/DocumentToLineStream.java
@@ -41,7 +41,7 @@ public class DocumentToLineStream extends SegmenterObjectStream<String, String>
// documents must be empty line terminated
if (!lines.get(lines.size() - 1).trim().isEmpty()) {
- lines = new ArrayList<String>(lines);
+ lines = new ArrayList<>(lines);
lines.add("");
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
index 770a698..744e134 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStream.java
@@ -42,12 +42,12 @@ public class OntoNotesNameSampleStream extends
private final Map<String, String> tokenConversionMap;
- private List<NameSample> nameSamples = new LinkedList<NameSample>();
+ private List<NameSample> nameSamples = new LinkedList<>();
public OntoNotesNameSampleStream(ObjectStream<String> samples) {
super(samples);
- Map<String, String> tokenConversionMap = new HashMap<String, String>();
+ Map<String, String> tokenConversionMap = new HashMap<>();
tokenConversionMap.put("-LRB-", "(");
tokenConversionMap.put("-RRB-", ")");
tokenConversionMap.put("-LSB-", "[");
@@ -107,8 +107,8 @@ public class OntoNotesNameSampleStream extends
String tokens[] = WhitespaceTokenizer.INSTANCE.tokenize(line);
- List<Span> entities = new LinkedList<Span>();
- List<String> cleanedTokens = new ArrayList<String>(tokens.length);
+ List<Span> entities = new LinkedList<>();
+ List<String> cleanedTokens = new ArrayList<>(tokens.length);
int tokenIndex = 0;
int entityBeginIndex = -1;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
index 88b1424..c0dfdbd 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesNameSampleStreamFactory.java
@@ -18,9 +18,7 @@
package opennlp.tools.formats.ontonotes;
import java.io.File;
-import java.io.FileFilter;
import java.nio.charset.Charset;
-
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.formats.AbstractSampleStreamFactory;
@@ -41,19 +39,15 @@ public class OntoNotesNameSampleStreamFactory extends
OntoNotesFormatParameters params = ArgumentParser.parse(args, OntoNotesFormatParameters.class);
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
- params.getOntoNotesDir()), new FileFilter() {
-
- public boolean accept(File file) {
- if (file.isFile()) {
- return file.getName().endsWith(".name");
- }
+ params.getOntoNotesDir()), file -> {
+ if (file.isFile()) {
+ return file.getName().endsWith(".name");
+ }
- return file.isDirectory();
- }
- }, true);
+ return file.isDirectory();
+ }, true);
- return new OntoNotesNameSampleStream(new FileToStringSampleStream(
- documentStream, Charset.forName("UTF-8")));
+ return new OntoNotesNameSampleStream(new FileToStringSampleStream(documentStream, Charset.forName("UTF-8")));
}
public static void registerFactory() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
index e77edcf..a99bc4e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ontonotes/OntoNotesParseSampleStreamFactory.java
@@ -18,9 +18,7 @@
package opennlp.tools.formats.ontonotes;
import java.io.File;
-import java.io.FileFilter;
import java.nio.charset.Charset;
-
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.StreamFactoryRegistry;
import opennlp.tools.formats.AbstractSampleStreamFactory;
@@ -31,7 +29,6 @@ import opennlp.tools.util.ObjectStream;
public class OntoNotesParseSampleStreamFactory extends AbstractSampleStreamFactory<Parse> {
-
protected OntoNotesParseSampleStreamFactory() {
super(OntoNotesFormatParameters.class);
}
@@ -41,16 +38,13 @@ public class OntoNotesParseSampleStreamFactory extends AbstractSampleStreamFacto
OntoNotesFormatParameters params = ArgumentParser.parse(args, OntoNotesFormatParameters.class);
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
- params.getOntoNotesDir()), new FileFilter() {
-
- public boolean accept(File file) {
- if (file.isFile()) {
- return file.getName().endsWith(".parse");
- }
+ params.getOntoNotesDir()), file -> {
+ if (file.isFile()) {
+ return file.getName().endsWith(".parse");
+ }
- return file.isDirectory();
- }
- }, true);
+ return file.isDirectory();
+ }, true);
// We need file to line here ... and that is probably best doen with the plain text stream
// lets copy it over here, refactor it, and then at some point we replace the current version
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
index 683b97c..6d8b51d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Lemmatize by simple dictionary lookup into a hashmap built from a file
@@ -36,7 +37,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
/**
* The hashmap containing the dictionary.
*/
- private final HashMap<List<String>, String> dictMap;
+ private final Map<List<String>, String> dictMap;
/**
* Construct a hashmap from the input tab separated dictionary.
@@ -66,7 +67,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
*
* @return dictMap the Map
*/
- public HashMap<List<String>, String> getDictMap() {
+ public Map<List<String>, String> getDictMap() {
return this.dictMap;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
index 05a5424..7f087b0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISTrainer.java
@@ -20,16 +20,12 @@
package opennlp.tools.ml.maxent;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
import opennlp.tools.ml.model.DataIndexer;
import opennlp.tools.ml.model.EvalParameters;
import opennlp.tools.ml.model.Event;
@@ -256,7 +252,7 @@ class GISTrainer {
modelExpects = new MutableContext[threads][];
- /************** Incorporate all of the needed info ******************/
+ /* Incorporate all of the needed info *****/
display("Incorporating indexed data for training... \n");
contexts = di.getContexts();
values = di.getValues();
@@ -387,7 +383,7 @@ class GISTrainer {
display("...done.\n");
- /***************** Find the parameters ************************/
+ /* Find the parameters *****/
if (threads == 1)
display("Computing model parameters ...\n");
else
@@ -395,7 +391,7 @@ class GISTrainer {
findParameters(iterations, correctionConstant);
- /*************** Create and return the model ******************/
+ /* Create and return the model ****/
// To be compatible with old models the correction constant is always 1
return new GISModel(params, predLabels, outcomeLabels, 1, evalParams.getCorrectionParam());
@@ -405,7 +401,7 @@ class GISTrainer {
private void findParameters(int iterations, double correctionConstant) {
int threads=modelExpects.length;
ExecutorService executor = Executors.newFixedThreadPool(threads);
- CompletionService<ModelExpactationComputeTask> completionService=new ExecutorCompletionService<GISTrainer.ModelExpactationComputeTask>(executor);
+ CompletionService<ModelExpactationComputeTask> completionService = new ExecutorCompletionService<>(executor);
double prevLL = 0.0;
double currLL;
display("Performing " + iterations + " iterations.\n");
@@ -571,7 +567,7 @@ class GISTrainer {
}
for (int i=0; i<numberOfThreads; i++) {
- ModelExpactationComputeTask finishedTask = null;
+ ModelExpactationComputeTask finishedTask;
try {
finishedTask = completionService.take().get();
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/model/GenericModelReader.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/GenericModelReader.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/GenericModelReader.java
index fc5da33..37c000d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/GenericModelReader.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/GenericModelReader.java
@@ -41,20 +41,21 @@ public class GenericModelReader extends AbstractModelReader {
public void checkModelType() throws IOException {
String modelType = readUTF();
- if (modelType.equals("Perceptron")) {
- delegateModelReader = new PerceptronModelReader(this.dataReader);
- }
- else if (modelType.equals("GIS")) {
- delegateModelReader = new GISModelReader(this.dataReader);
- }
- else if (modelType.equals("QN")) {
+ switch (modelType) {
+ case "Perceptron":
+ delegateModelReader = new PerceptronModelReader(this.dataReader);
+ break;
+ case "GIS":
+ delegateModelReader = new GISModelReader(this.dataReader);
+ break;
+ case "QN":
delegateModelReader = new QNModelReader(this.dataReader);
- }
- else if (modelType.equals("NaiveBayes")) {
+ break;
+ case "NaiveBayes":
delegateModelReader = new NaiveBayesModelReader(this.dataReader);
- }
- else {
- throw new IOException("Unknown model format: "+modelType);
+ break;
+ default:
+ throw new IOException("Unknown model format: " + modelType);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
index 8bb05d2..272f608 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
@@ -73,12 +73,12 @@ public class OnePassRealValueDataIndexer extends OnePassDataIndexer {
}
protected List<ComparableEvent> index(LinkedList<Event> events, Map<String,Integer> predicateIndex) {
- Map<String,Integer> omap = new HashMap<String,Integer>();
+ Map<String,Integer> omap = new HashMap<>();
int numEvents = events.size();
int outcomeCount = 0;
- List<ComparableEvent> eventsToCompare = new ArrayList<ComparableEvent>(numEvents);
- List<Integer> indexedContext = new ArrayList<Integer>();
+ List<ComparableEvent> eventsToCompare = new ArrayList<>(numEvents);
+ List<Integer> indexedContext = new ArrayList<>();
for (int eventIndex=0; eventIndex<numEvents; eventIndex++) {
Event ev = events.removeFirst();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/BinaryNaiveBayesModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/BinaryNaiveBayesModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/BinaryNaiveBayesModelWriter.java
index f91d640..a00e8e5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/BinaryNaiveBayesModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/BinaryNaiveBayesModelWriter.java
@@ -31,7 +31,7 @@ import opennlp.tools.ml.model.AbstractModel;
* Model writer that saves models in binary format.
*/
public class BinaryNaiveBayesModelWriter extends NaiveBayesModelWriter {
- DataOutputStream output;
+ private DataOutputStream output;
/**
* Constructor which takes a NaiveBayesModel and a File and prepares itself to
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
index b39b659..7547630 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesTrainer.java
@@ -137,7 +137,7 @@ public class NaiveBayesTrainer extends AbstractEventTrainer {
display("...done.\n");
- /*************** Create and return the model ******************/
+ /* Create and return the model ****/
return new NaiveBayesModel(finalParameters, predLabels, outcomeLabels);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/BinaryPerceptronModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/BinaryPerceptronModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/BinaryPerceptronModelWriter.java
index df13efe..8d0085b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/BinaryPerceptronModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/BinaryPerceptronModelWriter.java
@@ -31,7 +31,7 @@ import opennlp.tools.ml.model.AbstractModel;
* Model writer that saves models in binary format.
*/
public class BinaryPerceptronModelWriter extends PerceptronModelWriter {
- DataOutputStream output;
+ private DataOutputStream output;
/**
* Constructor which takes a GISModel and a File and prepares itself to
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java
index 274e7df..e958c3c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronModelWriter.java
@@ -40,7 +40,7 @@ public abstract class PerceptronModelWriter extends AbstractModelWriter {
protected Context[] PARAMS;
protected String[] OUTCOME_LABELS;
protected String[] PRED_LABELS;
- int numOutcomes;
+ private int numOutcomes;
public PerceptronModelWriter (AbstractModel model) {
@@ -99,15 +99,15 @@ public abstract class PerceptronModelWriter extends AbstractModelWriter {
protected List<List<ComparablePredicate>> computeOutcomePatterns(ComparablePredicate[] sorted) {
ComparablePredicate cp = sorted[0];
- List<List<ComparablePredicate>> outcomePatterns = new ArrayList<List<ComparablePredicate>>();
- List<ComparablePredicate> newGroup = new ArrayList<ComparablePredicate>();
+ List<List<ComparablePredicate>> outcomePatterns = new ArrayList<>();
+ List<ComparablePredicate> newGroup = new ArrayList<>();
for (ComparablePredicate predicate : sorted) {
if (cp.compareTo(predicate) == 0) {
newGroup.add(predicate);
} else {
cp = predicate;
outcomePatterns.add(newGroup);
- newGroup = new ArrayList<ComparablePredicate>();
+ newGroup = new ArrayList<>();
newGroup.add(predicate);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronTrainer.java
index d90d856..68883fa 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PerceptronTrainer.java
@@ -86,12 +86,7 @@ public class PerceptronTrainer extends AbstractEventTrainer {
public boolean isValid() {
String algorithmName = getAlgorithm();
-
- if (algorithmName != null && !(PERCEPTRON_VALUE.equals(algorithmName))) {
- return false;
- }
-
- return true;
+ return !(algorithmName != null && !(PERCEPTRON_VALUE.equals(algorithmName)));
}
public boolean isSortAndMerge() {
@@ -217,7 +212,7 @@ public class PerceptronTrainer extends AbstractEventTrainer {
display("...done.\n");
- /*************** Create and return the model ******************/
+ /* Create and return the model *************/
return new PerceptronModel(finalParameters, predLabels, outcomeLabels);
}
@@ -229,7 +224,7 @@ public class PerceptronTrainer extends AbstractEventTrainer {
for (int oi = 0; oi < numOutcomes; oi++)
allOutcomesPattern[oi] = oi;
- /** Stores the estimated parameter value of each predicate during iteration. */
+ /* Stores the estimated parameter value of each predicate during iteration. */
MutableContext[] params = new MutableContext[numPreds];
for (int pi = 0; pi < numPreds; pi++) {
params[pi] = new MutableContext(allOutcomesPattern,new double[numOutcomes]);
@@ -239,7 +234,7 @@ public class PerceptronTrainer extends AbstractEventTrainer {
EvalParameters evalParams = new EvalParameters(params,numOutcomes);
- /** Stores the sum of parameter values of each predicate over many iterations. */
+ /* Stores the sum of parameter values of each predicate over many iterations. */
MutableContext[] summedParams = new MutableContext[numPreds];
if (useAverage) {
for (int pi = 0; pi < numPreds; pi++) {
@@ -273,7 +268,7 @@ public class PerceptronTrainer extends AbstractEventTrainer {
for (int ei = 0; ei < numUniqueEvents; ei++) {
int targetOutcome = outcomeList[ei];
- for (int ni=0; ni<this.numTimesEventsSeen[ei]; ni++) {
+ for (int ni = 0; ni < this.numTimesEventsSeen[ei]; ni++) {
// Compute the model's prediction according to the current parameters.
double[] modelDistribution = new double[numOutcomes];
@@ -295,8 +290,8 @@ public class PerceptronTrainer extends AbstractEventTrainer {
params[pi].updateParameter(targetOutcome, stepsize);
params[pi].updateParameter(maxOutcome, -stepsize);
} else {
- params[pi].updateParameter(targetOutcome, stepsize*values[ei][ci]);
- params[pi].updateParameter(maxOutcome, -stepsize*values[ei][ci]);
+ params[pi].updateParameter(targetOutcome, stepsize * values[ei][ci]);
+ params[pi].updateParameter(maxOutcome, -stepsize * values[ei][ci]);
}
}
}
@@ -309,36 +304,28 @@ public class PerceptronTrainer extends AbstractEventTrainer {
// Calculate the training accuracy and display.
double trainingAccuracy = (double) numCorrect / numEvents;
- if (i < 10 || (i%10) == 0)
- display(". (" + numCorrect + "/" + numEvents+") " + trainingAccuracy + "\n");
+ if (i < 10 || (i % 10) == 0)
+ display(". (" + numCorrect + "/" + numEvents + ") " + trainingAccuracy + "\n");
// TODO: Make averaging configurable !!!
boolean doAveraging;
- if (useAverage && useSkippedlAveraging && (i < 20 || isPerfectSquare(i))) {
- doAveraging = true;
- }
- else if (useAverage) {
- doAveraging = true;
- }
- else {
- doAveraging = false;
- }
+ doAveraging = useAverage && useSkippedlAveraging && (i < 20 || isPerfectSquare(i)) || useAverage;
if (doAveraging) {
numTimesSummed++;
for (int pi = 0; pi < numPreds; pi++)
- for (int aoi=0;aoi<numOutcomes;aoi++)
+ for (int aoi = 0; aoi < numOutcomes; aoi++)
summedParams[pi].updateParameter(aoi, params[pi].getParameters()[aoi]);
}
// If the tolerance is greater than the difference between the
// current training accuracy and all of the previous three
// training accuracies, stop training.
- if (Math.abs(prevAccuracy1-trainingAccuracy) < tolerance
- && Math.abs(prevAccuracy2-trainingAccuracy) < tolerance
- && Math.abs(prevAccuracy3-trainingAccuracy) < tolerance) {
+ if (Math.abs(prevAccuracy1 - trainingAccuracy) < tolerance
+ && Math.abs(prevAccuracy2 - trainingAccuracy) < tolerance
+ && Math.abs(prevAccuracy3 - trainingAccuracy) < tolerance) {
display("Stopping: change in training set accuracy less than " + tolerance + "\n");
break;
}
@@ -419,7 +406,7 @@ public class PerceptronTrainer extends AbstractEventTrainer {
// See whether a number is a perfect square. Inefficient, but fine
// for our purposes.
- private final static boolean isPerfectSquare (int n) {
+ private static boolean isPerfectSquare (int n) {
int root = (int)Math.sqrt(n);
return root*root == n;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PlainTextPerceptronModelWriter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PlainTextPerceptronModelWriter.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PlainTextPerceptronModelWriter.java
index 0322cc5..ffde6c6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PlainTextPerceptronModelWriter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/PlainTextPerceptronModelWriter.java
@@ -21,20 +21,18 @@ package opennlp.tools.ml.perceptron;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.zip.GZIPOutputStream;
-
import opennlp.tools.ml.model.AbstractModel;
/**
* Model writer that saves models in plain text format.
*/
public class PlainTextPerceptronModelWriter extends PerceptronModelWriter {
- BufferedWriter output;
+ private BufferedWriter output;
/**
* Constructor which takes a PerceptronModel and a File and prepares itself to
@@ -45,7 +43,7 @@ public class PlainTextPerceptronModelWriter extends PerceptronModelWriter {
* @param f The File in which the model is to be persisted.
*/
public PlainTextPerceptronModelWriter (AbstractModel model, File f)
- throws IOException, FileNotFoundException {
+ throws IOException {
super(model);
if (f.getName().endsWith(".gz")) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java
index 43537c5..f14ebf1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/perceptron/SimplePerceptronSequenceTrainer.java
@@ -20,9 +20,10 @@
package opennlp.tools.ml.perceptron;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-
import opennlp.tools.ml.AbstractEventModelSequenceTrainer;
import opennlp.tools.ml.model.AbstractModel;
import opennlp.tools.ml.model.DataIndexer;
@@ -61,7 +62,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
private String[] outcomeLabels;
- double[] modelDistribution;
+ private double[] modelDistribution;
/** Stores the average parameter values of each predicate during iteration. */
private MutableContext[] averageParams;
@@ -75,13 +76,13 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
private MutableContext[] params;
private boolean useAverage;
private int[][][] updates;
- private int VALUE = 0;
- private int ITER = 1;
- private int EVENT = 2;
+ private static final int VALUE = 0;
+ private static final int ITER = 1;
+ private static final int EVENT = 2;
private int[] allOutcomesPattern;
private String[] predLabels;
- int numSequences;
+ private int numSequences;
public SimplePerceptronSequenceTrainer() {
}
@@ -94,12 +95,8 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
String algorithmName = getAlgorithm();
- if (algorithmName != null
- && !(PERCEPTRON_SEQUENCE_VALUE.equals(algorithmName))) {
- return false;
- }
-
- return true;
+ return !(algorithmName != null
+ && !(PERCEPTRON_SEQUENCE_VALUE.equals(algorithmName)));
}
public AbstractModel doTrain(SequenceStream events) throws IOException {
@@ -127,7 +124,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
outcomeList = di.getOutcomeList();
predLabels = di.getPredLabels();
- pmap = new HashMap<String, Integer>();
+ pmap = new HashMap<>();
for (int i = 0; i < predLabels.length; i++) {
pmap.put(predLabels[i], i);
@@ -139,7 +136,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
this.iterations = iterations;
outcomeLabels = di.getOutcomeLabels();
- omap = new HashMap<String,Integer>();
+ omap = new HashMap<>();
for (int oli=0;oli<outcomeLabels.length;oli++) {
omap.put(outcomeLabels[oli], oli);
}
@@ -180,7 +177,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
findParameters(iterations);
display("...done.\n");
- /*************** Create and return the model ******************/
+ /* Create and return the model ****/
String[] updatedPredLabels = predLabels;
/*
String[] updatedPredLabels = new String[pmap.size()];
@@ -225,9 +222,9 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
int numCorrect = 0;
int oei=0;
int si=0;
- Map<String,Float>[] featureCounts = new Map[numOutcomes];
+ List<Map<String,Float>> featureCounts = new ArrayList<>(numOutcomes);
for (int oi=0;oi<numOutcomes;oi++) {
- featureCounts[oi] = new HashMap<String,Float>();
+ featureCounts.add(new HashMap<>());
}
PerceptronModel model = new PerceptronModel(params,predLabels,pmap,outcomeLabels);
@@ -249,7 +246,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
}
if (update) {
for (int oi=0;oi<numOutcomes;oi++) {
- featureCounts[oi].clear();
+ featureCounts.get(oi).clear();
}
//System.err.print("train:");for (int ei=0;ei<events.length;ei++) {System.err.print(" "+events[ei].getOutcome());} System.err.println();
//training feature count computation
@@ -262,14 +259,14 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
if (values != null) {
value = values[ci];
}
- Float c = featureCounts[oi].get(contextStrings[ci]);
+ Float c = featureCounts.get(oi).get(contextStrings[ci]);
if (c == null) {
c = value;
}
else {
c+=value;
}
- featureCounts[oi].put(contextStrings[ci], c);
+ featureCounts.get(oi).put(contextStrings[ci], c);
}
}
//evaluation feature count computation
@@ -283,7 +280,7 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
if (values != null) {
value = values[ci];
}
- Float c = featureCounts[oi].get(contextStrings[ci]);
+ Float c = featureCounts.get(oi).get(contextStrings[ci]);
if (c == null) {
c = -1*value;
}
@@ -291,19 +288,19 @@ public class SimplePerceptronSequenceTrainer extends AbstractEventModelSequenceT
c-=value;
}
if (c == 0f) {
- featureCounts[oi].remove(contextStrings[ci]);
+ featureCounts.get(oi).remove(contextStrings[ci]);
}
else {
- featureCounts[oi].put(contextStrings[ci], c);
+ featureCounts.get(oi).put(contextStrings[ci], c);
}
}
}
for (int oi=0;oi<numOutcomes;oi++) {
- for (String feature : featureCounts[oi].keySet()) {
+ for (String feature : featureCounts.get(oi).keySet()) {
int pi = pmap.get(feature);
if (pi != -1) {
//System.err.println(si+" "+outcomeLabels[oi]+" "+feature+" "+featureCounts[oi].get(feature));
- params[pi].updateParameter(oi, featureCounts[oi].get(feature));
+ params[pi].updateParameter(oi, featureCounts.get(oi).get(feature));
if (useAverage) {
if (updates[pi][oi][VALUE] != 0) {
averageParams[pi].updateParameter(oi,updates[pi][oi][VALUE]*(numSequences*(iteration-updates[pi][oi][ITER])+(si-updates[pi][oi][EVENT])));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
index 00e001c..ff8c143 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameFinderME.java
@@ -45,7 +45,6 @@ import opennlp.tools.util.featuregen.AdaptiveFeatureGenerator;
import opennlp.tools.util.featuregen.AdditionalContextFeatureGenerator;
import opennlp.tools.util.featuregen.BigramNameFeatureGenerator;
import opennlp.tools.util.featuregen.CachedFeatureGenerator;
-import opennlp.tools.util.featuregen.FeatureGeneratorResourceProvider;
import opennlp.tools.util.featuregen.GeneratorFactory;
import opennlp.tools.util.featuregen.OutcomePriorFeatureGenerator;
import opennlp.tools.util.featuregen.PreviousMapFeatureGenerator;
@@ -114,14 +113,11 @@ public class NameFinderME implements TokenNameFinder {
if (generatorDescriptor != null) {
featureGenerator = GeneratorFactory.create(new ByteArrayInputStream(
- generatorDescriptor), new FeatureGeneratorResourceProvider() {
-
- public Object getResource(String key) {
- if (resources != null) {
- return resources.get(key);
- }
- return null;
+ generatorDescriptor), key -> {
+ if (resources != null) {
+ return resources.get(key);
}
+ return null;
});
} else {
featureGenerator = null;
@@ -440,7 +436,7 @@ public class NameFinderME implements TokenNameFinder {
*/
public static Span[] dropOverlappingSpans(Span spans[]) {
- List<Span> sortedSpans = new ArrayList<Span>(spans.length);
+ List<Span> sortedSpans = new ArrayList<>(spans.length);
Collections.addAll(sortedSpans, spans);
Collections.sort(sortedSpans);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
index 0e597e0..6d912da 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
@@ -25,12 +25,10 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.NoSuchElementException;
-
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.dictionary.serializer.Attributes;
import opennlp.tools.dictionary.serializer.DictionarySerializer;
import opennlp.tools.dictionary.serializer.Entry;
-import opennlp.tools.dictionary.serializer.EntryInserter;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.StringList;
import opennlp.tools.util.StringUtil;
@@ -57,11 +55,9 @@ public class NGramModel implements Iterable<StringList>{
*
* @param in the serialized model stream
* @throws IOException
- * @throws InvalidFormatException
*/
- public NGramModel(InputStream in) throws IOException, InvalidFormatException {
- DictionarySerializer.create(in, new EntryInserter() {
- public void insert(Entry entry) throws InvalidFormatException {
+ public NGramModel(InputStream in) throws IOException {
+ DictionarySerializer.create(in, entry -> {
int count;
String countValueString = null;
@@ -82,8 +78,7 @@ public class NGramModel implements Iterable<StringList>{
add(entry.getTokens());
setCount(entry.getTokens(), count);
- }
- });
+ });
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
index ff4bea8..cc8eab5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
@@ -349,7 +349,7 @@ public abstract class AbstractBottomUpParser implements Parser {
return new Parse[] {completeParses.first()};
}
else {
- List<Parse> topParses = new ArrayList<Parse>(numParses);
+ List<Parse> topParses = new ArrayList<>(numParses);
while(!completeParses.isEmpty() && topParses.size() < numParses) {
Parse tp = completeParses.extract();
topParses.add(tp);
@@ -460,9 +460,6 @@ public abstract class AbstractBottomUpParser implements Parser {
words[i] = children[i].getCoveredText();
}
Sequence[] ts = tagger.topKSequences(words);
-// if (ts.length == 0) {
-// System.err.println("no tag sequence");
-// }
Parse[] newParses = new Parse[ts.length];
for (int i = 0; i < ts.length; i++) {
String[] tags = ts[i].getOutcomes().toArray(new String[words.length]);
@@ -475,7 +472,6 @@ public abstract class AbstractBottomUpParser implements Parser {
double prob = probs[j];
newParses[i].insert(new Parse(word.getText(), word.getSpan(), tags[j], prob,j));
newParses[i].addProb(Math.log(prob));
- //newParses[i].show();
}
}
return newParses;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
index 09d3b0f..85de098 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
@@ -118,7 +118,6 @@ public abstract class AbstractContextGenerator {
if (punct1s != null) {
for (Iterator<Parse> pi = punct1s.iterator();pi.hasNext();) {
Parse p = pi.next();
-// String punct = punct(p,c1.index);
String punctbo = punctbo(p,c1.index <= 0 ? c1.index -1 : c1.index);
//punctbo(1);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
index d3d8303..b3de9b2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
@@ -115,7 +115,7 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
String cacheKey = i +t_2+t1+t0+t1+t2+p_2+p_1;
if (contextsCache!= null) {
if (wordsKey == words) {
- String[] contexts = (String[]) contextsCache.get(cacheKey);
+ String[] contexts = contextsCache.get(cacheKey);
if (contexts != null) {
return contexts;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
index 9bce4cc..93b23ad 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/AttachContextGenerator.java
@@ -60,8 +60,7 @@ public class AttachContextGenerator extends AbstractContextGenerator {
* @return A set of contextual features about this attachment.
*/
public String[] getContext(Parse[] constituents, int index, List<Parse> rightFrontier, int rfi) {
- List<String> features = new ArrayList<String>(100);
- int nodeDistance = rfi;
+ List<String> features = new ArrayList<>(100);
Parse fn = rightFrontier.get(rfi);
Parse fp = null;
if (rfi+1 < rightFrontier.size()) {
@@ -147,9 +146,9 @@ public class AttachContextGenerator extends AbstractContextGenerator {
*/
int headDistance = (p0.getHeadIndex()-fn.getHeadIndex());
features.add("hd="+headDistance);
- features.add("nd="+nodeDistance);
+ features.add("nd="+ rfi);
- features.add("nd="+p0.getType()+"."+nodeDistance);
+ features.add("nd="+p0.getType()+"."+ rfi);
features.add("hd="+p0.getType()+"."+headDistance);
//features.add("fs="+rightFrontier.size());
//paired punct features
@@ -158,9 +157,6 @@ public class AttachContextGenerator extends AbstractContextGenerator {
features.add("quotematch");//? not generating feature correctly
}
- else {
- //features.add("noquotematch");
- }
}
return features.toArray(new String[features.size()]);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
index 8677171..589a1dc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/BuildContextGenerator.java
@@ -115,7 +115,7 @@ public class BuildContextGenerator extends AbstractContextGenerator {
Cons c1 = new Cons(consp1,consbop1,1,true);
Cons c2 = new Cons(consp2,consbop2,2,true);
- List<String> features = new ArrayList<String>();
+ List<String> features = new ArrayList<>();
features.add("default");
//unigrams
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
index 694d65d..de6824d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/CheckContextGenerator.java
@@ -42,7 +42,7 @@ public class CheckContextGenerator extends AbstractContextGenerator {
}
public String[] getContext(Parse parent, Parse[] constituents, int index, boolean trimFrontier) {
- List<String> features = new ArrayList<String>(100);
+ List<String> features = new ArrayList<>(100);
//default
features.add("default");
Parse[] children = Parser.collapsePunctuation(parent.getChildren(),punctSet);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
index e2c62d7..85d6cf9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
@@ -140,7 +140,7 @@ public class Parser extends AbstractBottomUpParser {
* @return The right frontier of the specified parse tree.
*/
public static List<Parse> getRightFrontier(Parse root,Set<String> punctSet) {
- List<Parse> rf = new LinkedList<Parse>();
+ List<Parse> rf = new LinkedList<>();
Parse top;
if (AbstractBottomUpParser.TOP_NODE.equals(root.getType()) ||
AbstractBottomUpParser.INC_NODE.equals(root.getType())) {
@@ -154,7 +154,7 @@ public class Parser extends AbstractBottomUpParser {
Parse[] kids = top.getChildren();
top = kids[kids.length-1];
}
- return new ArrayList<Parse>(rf);
+ return new ArrayList<>(rf);
}
private void setBuilt(Parse p) {
@@ -193,22 +193,12 @@ public class Parser extends AbstractBottomUpParser {
private boolean isBuilt(Parse p) {
String l = p.getLabel();
- if (l == null) {
- return false;
- }
- else {
- return l.startsWith(Parser.BUILT);
- }
+ return l != null && l.startsWith(Parser.BUILT);
}
private boolean isComplete(Parse p) {
String l = p.getLabel();
- if (l == null) {
- return false;
- }
- else {
- return l.endsWith(Parser.COMPLETE);
- }
+ return l != null && l.endsWith(Parser.COMPLETE);
}
@Override
@@ -226,9 +216,9 @@ public class Parser extends AbstractBottomUpParser {
@Override
protected Parse[] advanceParses(Parse p, double probMass) {
double q = 1 - probMass;
- /** The index of the node which will be labeled in this iteration of advancing the parse. */
+ /* The index of the node which will be labeled in this iteration of advancing the parse. */
int advanceNodeIndex;
- /** The node which will be labeled in this iteration of advancing the parse. */
+ /* The node which will be labeled in this iteration of advancing the parse. */
Parse advanceNode=null;
Parse[] originalChildren = p.getChildren();
Parse[] children = collapsePunctuation(originalChildren,punctSet);
@@ -254,7 +244,7 @@ public class Parser extends AbstractBottomUpParser {
}
int originalZeroIndex = mapParseIndex(0,children,originalChildren);
int originalAdvanceIndex = mapParseIndex(advanceNodeIndex,children,originalChildren);
- List<Parse> newParsesList = new ArrayList<Parse>();
+ List<Parse> newParsesList = new ArrayList<>();
//call build model
buildModel.eval(buildContextGenerator.getContext(children, advanceNodeIndex), bprobs);
double doneProb = bprobs[doneIndex];
@@ -262,7 +252,7 @@ public class Parser extends AbstractBottomUpParser {
if (1-doneProb > q) {
double bprobSum = 0;
while (bprobSum < probMass) {
- /** The largest unadvanced labeling. */
+ /* The largest unadvanced labeling. */
int max = 0;
for (int pi = 1; pi < bprobs.length; pi++) { //for each build outcome
if (bprobs[pi] > bprobs[max]) {
@@ -434,7 +424,7 @@ public class Parser extends AbstractBottomUpParser {
ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams)
throws IOException {
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
System.err.println("Building dictionary");
Dictionary mdict = buildDictionary(parseSamples, rules, mlParams);
@@ -457,7 +447,7 @@ public class Parser extends AbstractBottomUpParser {
System.err.println("Training builder");
ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules,
ParserEventTypeEnum.BUILD, mdict);
- Map<String, String> buildReportMap = new HashMap<String, String>();
+ Map<String, String> buildReportMap = new HashMap<>();
EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getSettings("build"), buildReportMap);
MaxentModel buildModel = buildTrainer.train(bes);
@@ -469,7 +459,7 @@ public class Parser extends AbstractBottomUpParser {
System.err.println("Training checker");
ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules,
ParserEventTypeEnum.CHECK);
- Map<String, String> checkReportMap = new HashMap<String, String>();
+ Map<String, String> checkReportMap = new HashMap<>();
EventTrainer checkTrainer = TrainerFactory.getEventTrainer(mlParams.getSettings("check"), checkReportMap);
MaxentModel checkModel = checkTrainer.train(kes);
@@ -481,7 +471,7 @@ public class Parser extends AbstractBottomUpParser {
System.err.println("Training attacher");
ObjectStream<Event> attachEvents = new ParserEventStream(parseSamples, rules,
ParserEventTypeEnum.ATTACH);
- Map<String, String> attachReportMap = new HashMap<String, String>();
+ Map<String, String> attachReportMap = new HashMap<>();
EventTrainer attachTrainer = TrainerFactory.getEventTrainer(mlParams.getSettings("attach"), attachReportMap);
MaxentModel attachModel = attachTrainer.train(attachEvents);
opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest(manifestInfoEntries, attachReportMap, "attach");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
index c693ef6..5949414 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/ParserEventStream.java
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
import opennlp.tools.cmdline.SystemInputStreamFactory;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.ml.maxent.io.SuffixSensitiveGISModelReader;
@@ -37,7 +36,6 @@ import opennlp.tools.parser.HeadRules;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.ParseSampleStream;
import opennlp.tools.parser.ParserEventTypeEnum;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -73,7 +71,7 @@ public class ParserEventStream extends AbstractParserEventStream {
* @return a set of parent nodes.
*/
private Map<Parse, Integer> getNonAdjoinedParent(Parse node) {
- Map<Parse, Integer> parents = new HashMap<Parse, Integer>();
+ Map<Parse, Integer> parents = new HashMap<>();
Parse parent = node.getParent();
int index = indexOf(node,parent);
parents.put(parent, index);
@@ -130,11 +128,11 @@ public class ParserEventStream extends AbstractParserEventStream {
@Override
protected void addParseEvents(List<Event> parseEvents, Parse[] chunks) {
- /** Frontier nodes built from node in a completed parse. Specifically,
+ /* Frontier nodes built from node in a completed parse. Specifically,
* they have all their children regardless of the stage of parsing.*/
- List<Parse> rightFrontier = new ArrayList<Parse>();
- List<Parse> builtNodes = new ArrayList<Parse>();
- /** Nodes which characterize what the parse looks like to the parser as its being built.
+ List<Parse> rightFrontier = new ArrayList<>();
+ List<Parse> builtNodes = new ArrayList<>();
+ /* Nodes which characterize what the parse looks like to the parser as its being built.
* Specifically, these nodes don't have all their children attached like the parents of
* the chunk nodes do.*/
Parse[] currentChunks = new Parse[chunks.length];
@@ -200,7 +198,7 @@ public class ParserEventStream extends AbstractParserEventStream {
}
//attach node
String attachType = null;
- /** Node selected for attachment. */
+ /* Node selected for attachment. */
Parse attachNode = null;
int attachNodeIndex = -1;
if (ci == 0){
@@ -208,7 +206,7 @@ public class ParserEventStream extends AbstractParserEventStream {
top.insert(currentChunks[ci]);
}
else {
- /** Right frontier consisting of partially-built nodes based on current state of the parse.*/
+ /* Right frontier consisting of partially-built nodes based on current state of the parse.*/
List<Parse> currentRightFrontier = Parser.getRightFrontier(currentChunks[0],punctSet);
if (currentRightFrontier.size() != rightFrontier.size()) {
System.err.println("fontiers mis-aligned: "+currentRightFrontier.size()+" != "+rightFrontier.size()+" "+currentRightFrontier+" "+rightFrontier);
@@ -333,7 +331,7 @@ public class ParserEventStream extends AbstractParserEventStream {
}
}
- public static void main(String[] args) throws java.io.IOException, InvalidFormatException {
+ public static void main(String[] args) throws java.io.IOException {
if (args.length == 0) {
System.err.println("Usage ParserEventStream -[tag|chunk|build|attach] [-fun] [-dict dictionary] [-model model] head_rules < parses");
System.exit(1);
@@ -345,35 +343,36 @@ public class ParserEventStream extends AbstractParserEventStream {
AbstractModel model = null;
while (ai < args.length && args[ai].startsWith("-")) {
- if (args[ai].equals("-build")) {
- etype = ParserEventTypeEnum.BUILD;
- }
- else if (args[ai].equals("-attach")) {
- etype = ParserEventTypeEnum.ATTACH;
- }
- else if (args[ai].equals("-chunk")) {
- etype = ParserEventTypeEnum.CHUNK;
- }
- else if (args[ai].equals("-check")) {
- etype = ParserEventTypeEnum.CHECK;
- }
- else if (args[ai].equals("-tag")) {
- etype = ParserEventTypeEnum.TAG;
- }
- else if (args[ai].equals("-fun")) {
- fun = true;
- }
- else if (args[ai].equals("-dict")) {
- ai++;
- dict = new Dictionary(new FileInputStream(args[ai]));
- }
- else if (args[ai].equals("-model")) {
- ai++;
- model = (new SuffixSensitiveGISModelReader(new File(args[ai]))).getModel();
- }
- else {
- System.err.println("Invalid option " + args[ai]);
- System.exit(1);
+ switch (args[ai]) {
+ case "-build":
+ etype = ParserEventTypeEnum.BUILD;
+ break;
+ case "-attach":
+ etype = ParserEventTypeEnum.ATTACH;
+ break;
+ case "-chunk":
+ etype = ParserEventTypeEnum.CHUNK;
+ break;
+ case "-check":
+ etype = ParserEventTypeEnum.CHECK;
+ break;
+ case "-tag":
+ etype = ParserEventTypeEnum.TAG;
+ break;
+ case "-fun":
+ fun = true;
+ break;
+ case "-dict":
+ ai++;
+ dict = new Dictionary(new FileInputStream(args[ai]));
+ break;
+ case "-model":
+ ai++;
+ model = (new SuffixSensitiveGISModelReader(new File(args[ai]))).getModel();
+ break;
+ default:
+ System.err.println("Invalid option " + args[ai]);
+ System.exit(1);
}
ai++;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
index 34b9f79..21522a5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSModel.java
@@ -43,7 +43,7 @@ public final class POSModel extends BaseModel {
private static final String COMPONENT_NAME = "POSTaggerME";
- public static final String POS_MODEL_ENTRY_NAME = "pos.model";
+ static final String POS_MODEL_ENTRY_NAME = "pos.model";
public POSModel(String languageCode, SequenceClassificationModel<String> posModel,
Map<String, String> manifestInfoEntries, POSTaggerFactory posFactory) {
@@ -78,15 +78,15 @@ public final class POSModel extends BaseModel {
checkArtifactMap();
}
- public POSModel(InputStream in) throws IOException, InvalidFormatException {
+ public POSModel(InputStream in) throws IOException {
super(COMPONENT_NAME, in);
}
- public POSModel(File modelFile) throws IOException, InvalidFormatException {
+ public POSModel(File modelFile) throws IOException {
super(COMPONENT_NAME, modelFile);
}
- public POSModel(URL modelURL) throws IOException, InvalidFormatException {
+ public POSModel(URL modelURL) throws IOException {
super(COMPONENT_NAME, modelURL);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
index e4c1c1b..366a91a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
@@ -24,9 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.StringTokenizer;
import java.util.concurrent.atomic.AtomicInteger;
-
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.EventModelSequenceTrainer;
@@ -45,7 +43,6 @@ import opennlp.tools.util.StringList;
import opennlp.tools.util.StringUtil;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.featuregen.StringPattern;
-import opennlp.tools.util.model.ModelType;
/**
* A part-of-speech tagger that uses maximum entropy. Tries to predict whether
@@ -117,7 +114,7 @@ public class POSTaggerME implements POSTagger {
this.model = model.getPosSequenceModel();
}
else {
- this.model = new opennlp.tools.ml.BeamSearch<String>(beamSize,
+ this.model = new opennlp.tools.ml.BeamSearch<>(beamSize,
model.getPosModel(), 0);
}
@@ -127,7 +124,7 @@ public class POSTaggerME implements POSTagger {
* Retrieves an array of all possible part-of-speech tags from the
* tagger.
*
- * @return
+ * @return String[]
*/
public String[] getAllPosTags() {
return model.getOutcomes();
@@ -237,7 +234,7 @@ public class POSTaggerME implements POSTagger {
POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator();
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
TrainerType trainerType = TrainerFactory.getTrainerType(trainParams.getSettings());
@@ -302,7 +299,7 @@ public class POSTaggerME implements POSTagger {
// the data structure will store the word, the tag, and the number of
// occurrences
- Map<String, Map<String, AtomicInteger>> newEntries = new HashMap<String, Map<String, AtomicInteger>>();
+ Map<String, Map<String, AtomicInteger>> newEntries = new HashMap<>();
POSSample sample;
while ((sample = samples.read()) != null) {
String[] words = sample.getSentence();
@@ -319,7 +316,7 @@ public class POSTaggerME implements POSTagger {
}
if (!newEntries.containsKey(word)) {
- newEntries.put(word, new HashMap<String, AtomicInteger>());
+ newEntries.put(word, new HashMap<>());
}
String[] dictTags = dict.getTags(word);
@@ -346,7 +343,7 @@ public class POSTaggerME implements POSTagger {
// add it to the dictionary
for (Entry<String, Map<String, AtomicInteger>> wordEntry : newEntries
.entrySet()) {
- List<String> tagsForWord = new ArrayList<String>();
+ List<String> tagsForWord = new ArrayList<>();
for (Entry<String, AtomicInteger> entry : wordEntry.getValue().entrySet()) {
if (entry.getValue().get() >= cutoff) {
tagsForWord.add(entry.getKey());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java b/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
index 873c2c4..499aa8f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/BaseToolFactory.java
@@ -63,7 +63,7 @@ public abstract class BaseToolFactory {
*/
@SuppressWarnings("rawtypes")
public Map<String, ArtifactSerializer> createArtifactSerializersMap() {
- return new HashMap<String, ArtifactSerializer>();
+ return new HashMap<>();
}
/**
@@ -75,7 +75,7 @@ public abstract class BaseToolFactory {
* populated by sub-classes.
*/
public Map<String, Object> createArtifactMap() {
- return new HashMap<String, Object>();
+ return new HashMap<>();
}
/**
@@ -84,7 +84,7 @@ public abstract class BaseToolFactory {
* @return the manifest entries to added to the model manifest
*/
public Map<String, String> createManifestEntries() {
- return new HashMap<String, String>();
+ return new HashMap<>();
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
index f81cd36..737fae3 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4NameFinderEval.java
@@ -18,13 +18,8 @@
package opennlp.tools.eval;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
import java.nio.charset.Charset;
-
-import org.junit.Assert;
-import org.junit.Test;
-
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.formats.ontonotes.OntoNotesNameSampleStream;
@@ -35,6 +30,8 @@ import opennlp.tools.namefind.TokenNameFinderFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
+import org.junit.Assert;
+import org.junit.Test;
public class OntoNotes4NameFinderEval {
@@ -42,16 +39,13 @@ public class OntoNotes4NameFinderEval {
throws IOException {
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
- EvalUtil.getOpennlpDataDir(), "ontonotes4/data/files/data/english"), new FileFilter() {
-
- public boolean accept(File file) {
- if (file.isFile()) {
- return file.getName().endsWith(".name");
- }
+ EvalUtil.getOpennlpDataDir(), "ontonotes4/data/files/data/english"), file -> {
+ if (file.isFile()) {
+ return file.getName().endsWith(".name");
+ }
- return file.isDirectory();
- }
- }, true);
+ return file.isDirectory();
+ }, true);
ObjectStream<NameSample> samples = new OntoNotesNameSampleStream(new FileToStringSampleStream(
documentStream, Charset.forName("UTF-8")));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
index 84185e0..087ab7e 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4ParserEval.java
@@ -18,15 +18,10 @@
package opennlp.tools.eval;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
-
-import org.junit.Assert;
-import org.junit.Test;
-
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.formats.ontonotes.DocumentToLineStream;
@@ -38,6 +33,8 @@ import opennlp.tools.parser.lang.en.HeadRulesTest;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
+import org.junit.Assert;
+import org.junit.Test;
public class OntoNotes4ParserEval {
@@ -45,16 +42,13 @@ public class OntoNotes4ParserEval {
throws IOException {
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
- EvalUtil.getOpennlpDataDir(), "ontonotes4/data/files/data/english"), new FileFilter() {
-
- public boolean accept(File file) {
- if (file.isFile()) {
- return file.getName().endsWith(".parse");
- }
+ EvalUtil.getOpennlpDataDir(), "ontonotes4/data/files/data/english"), file -> {
+ if (file.isFile()) {
+ return file.getName().endsWith(".parse");
+ }
- return file.isDirectory();
- }
- }, true);
+ return file.isDirectory();
+ }, true);
OntoNotesParseSampleStream samples = new OntoNotesParseSampleStream(
new DocumentToLineStream(new FileToStringSampleStream(
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
index ca1676a..fb25836 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/OntoNotes4PosTaggerEval.java
@@ -18,13 +18,8 @@
package opennlp.tools.eval;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
import java.nio.charset.Charset;
-
-import org.junit.Assert;
-import org.junit.Test;
-
import opennlp.tools.formats.DirectorySampleStream;
import opennlp.tools.formats.convert.FileToStringSampleStream;
import opennlp.tools.formats.convert.ParseToPOSSampleStream;
@@ -35,6 +30,8 @@ import opennlp.tools.postag.POSTaggerFactory;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.model.ModelUtil;
+import org.junit.Assert;
+import org.junit.Test;
public class OntoNotes4PosTaggerEval {
@@ -42,16 +39,13 @@ public class OntoNotes4PosTaggerEval {
throws IOException {
ObjectStream<File> documentStream = new DirectorySampleStream(new File(
- EvalUtil.getOpennlpDataDir(), "ontonotes4/data/files/data/english"), new FileFilter() {
-
- public boolean accept(File file) {
- if (file.isFile()) {
- return file.getName().endsWith(".parse");
- }
+ EvalUtil.getOpennlpDataDir(), "ontonotes4/data/files/data/english"), file -> {
+ if (file.isFile()) {
+ return file.getName().endsWith(".parse");
+ }
- return file.isDirectory();
- }
- }, true);
+ return file.isDirectory();
+ }, true);
ParseToPOSSampleStream samples = new ParseToPOSSampleStream(new OntoNotesParseSampleStream(
new DocumentToLineStream(
http://git-wip-us.apache.org/repos/asf/opennlp/blob/9cb610bc/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
index 2f72124..ec76565 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
@@ -18,14 +18,10 @@
package opennlp.tools.postag;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-
-import org.junit.Test;
-
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.formats.ResourceAsStreamFactory;
import opennlp.tools.postag.DummyPOSTaggerFactory.DummyPOSContextGenerator;
@@ -37,7 +33,9 @@ import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.TrainingParameters;
-import opennlp.tools.util.model.ModelType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
/**
* Tests for the {@link POSTaggerFactory} class.
@@ -53,7 +51,7 @@ public class POSTaggerFactoryTest {
return new WordTagSampleStream(new PlainTextByLineStream(in, UTF_8));
}
- static POSModel trainPOSModel(ModelType type, POSTaggerFactory factory)
+ private static POSModel trainPOSModel(POSTaggerFactory factory)
throws IOException {
return POSTaggerME.train("en", createSampleStream(),
TrainingParameters.defaultParams(), factory);
@@ -66,8 +64,8 @@ public class POSTaggerFactoryTest {
.getResourceAsStream("TagDictionaryCaseSensitive.xml")));
Dictionary dic = POSTaggerME.buildNGramDictionary(createSampleStream(), 0);
- POSModel posModel = trainPOSModel(ModelType.MAXENT,
- new DummyPOSTaggerFactory(dic, posDict));
+ POSModel posModel = trainPOSModel(
+ new DummyPOSTaggerFactory(dic, posDict));
POSTaggerFactory factory = posModel.getFactory();
assertTrue(factory.getTagDictionary() instanceof DummyPOSDictionary);
@@ -84,7 +82,7 @@ public class POSTaggerFactoryTest {
assertTrue(factory.getTagDictionary() instanceof DummyPOSDictionary);
assertTrue(factory.getPOSContextGenerator() instanceof DummyPOSContextGenerator);
assertTrue(factory.getSequenceValidator() instanceof DummyPOSSequenceValidator);
- assertTrue(factory.getDictionary() instanceof Dictionary);
+ assertTrue(factory.getDictionary() != null);
}
@Test
@@ -93,14 +91,14 @@ public class POSTaggerFactoryTest {
.getResourceAsStream("TagDictionaryCaseSensitive.xml"));
Dictionary dic = POSTaggerME.buildNGramDictionary(createSampleStream(), 0);
- POSModel posModel = trainPOSModel(ModelType.MAXENT,
- new POSTaggerFactory(dic, posDict));
+ POSModel posModel = trainPOSModel(
+ new POSTaggerFactory(dic, posDict));
POSTaggerFactory factory = posModel.getFactory();
assertTrue(factory.getTagDictionary() instanceof POSDictionary);
- assertTrue(factory.getPOSContextGenerator() instanceof POSContextGenerator);
+ assertTrue(factory.getPOSContextGenerator() != null);
assertTrue(factory.getSequenceValidator() instanceof DefaultPOSSequenceValidator);
- assertTrue(factory.getDictionary() instanceof Dictionary);
+ assertTrue(factory.getDictionary() != null);
ByteArrayOutputStream out = new ByteArrayOutputStream();
posModel.serialize(out);
@@ -110,9 +108,9 @@ public class POSTaggerFactoryTest {
factory = fromSerialized.getFactory();
assertTrue(factory.getTagDictionary() instanceof POSDictionary);
- assertTrue(factory.getPOSContextGenerator() instanceof POSContextGenerator);
+ assertTrue(factory.getPOSContextGenerator() != null);
assertTrue(factory.getSequenceValidator() instanceof DefaultPOSSequenceValidator);
- assertTrue(factory.getDictionary() instanceof Dictionary);
+ assertTrue(factory.getDictionary() != null);
}
@Test(expected = InvalidFormatException.class)
[13/50] [abbrv] opennlp git commit: Remove unnecessary casts
Posted by jo...@apache.org.
Remove unnecessary casts
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/d79b97cc
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/d79b97cc
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/d79b97cc
Branch: refs/heads/889
Commit: d79b97cc662ce0428d1823cfa5afe7988a3419e6
Parents: 34af4ae
Author: J�rn Kottmann <jo...@apache.org>
Authored: Fri Oct 28 18:27:24 2016 +0200
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:35:01 2016 +0100
----------------------------------------------------------------------
.../cmdline/doccat/DoccatFineGrainedReportListener.java | 8 ++++----
.../lemmatizer/LemmatizerFineGrainedReportListener.java | 8 ++++----
.../postag/POSTaggerFineGrainedReportListener.java | 8 ++++----
.../opennlp/tools/languagemodel/NGramLanguageModel.java | 2 +-
.../opennlp/tools/ml/maxent/BasicContextGenerator.java | 2 +-
.../src/main/java/opennlp/tools/ml/maxent/GISModel.java | 2 +-
.../tools/ml/model/OnePassRealValueDataIndexer.java | 2 +-
.../main/java/opennlp/tools/parser/chunking/Parser.java | 2 +-
.../main/java/opennlp/tools/parser/treeinsert/Parser.java | 2 +-
.../main/java/opennlp/uima/chunker/ChunkerTrainer.java | 2 +-
.../java/opennlp/uima/doccat/DocumentCategorizer.java | 6 +++---
.../java/opennlp/uima/namefind/AbstractNameFinder.java | 10 +++++-----
.../java/opennlp/uima/normalizer/StringDictionary.java | 2 +-
.../src/main/java/opennlp/uima/parser/Parser.java | 5 ++---
.../main/java/opennlp/uima/postag/POSTaggerTrainer.java | 2 +-
.../java/opennlp/uima/util/AnnotationComboIterator.java | 10 +++++-----
16 files changed, 36 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
index 1f61b2c..932bd6f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/doccat/DoccatFineGrainedReportListener.java
@@ -625,13 +625,13 @@ public class DoccatFineGrainedReportListener implements DoccatEvaluationMonitor
for (String ref : tagset) {
int column = 0;
for (String pred : tagset) {
- matrix[line][column] = (double) (data.get(ref) != null ? data
- .get(ref).getValue(pred) : 0);
+ matrix[line][column] = data.get(ref) != null ? data
+ .get(ref).getValue(pred) : 0;
column++;
}
// set accuracy
- matrix[line][column] = (double) (data.get(ref) != null ? data.get(ref)
- .getAccuracy() : 0);
+ matrix[line][column] = data.get(ref) != null ? data.get(ref)
+ .getAccuracy() : 0;
line++;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
index 1818db1..da7013d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/lemmatizer/LemmatizerFineGrainedReportListener.java
@@ -754,13 +754,13 @@ public class LemmatizerFineGrainedReportListener
for (String ref : tagset) {
int column = 0;
for (String pred : tagset) {
- matrix[line][column] = (double) (data.get(ref) != null
- ? data.get(ref).getValue(pred) : 0);
+ matrix[line][column] = data.get(ref) != null
+ ? data.get(ref).getValue(pred) : 0;
column++;
}
// set accuracy
- matrix[line][column] = (double) (data.get(ref) != null
- ? data.get(ref).getAccuracy() : 0);
+ matrix[line][column] = data.get(ref) != null
+ ? data.get(ref).getAccuracy() : 0;
line++;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
index 743d00f..369f353 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerFineGrainedReportListener.java
@@ -761,13 +761,13 @@ public class POSTaggerFineGrainedReportListener implements
for (String ref : tagset) {
int column = 0;
for (String pred : tagset) {
- matrix[line][column] = (double) (data.get(ref) != null ? data
- .get(ref).getValue(pred) : 0);
+ matrix[line][column] = data.get(ref) != null ? data
+ .get(ref).getValue(pred) : 0;
column++;
}
// set accuracy
- matrix[line][column] = (double) (data.get(ref) != null ? data.get(ref)
- .getAccuracy() : 0);
+ matrix[line][column] = data.get(ref) != null ? data.get(ref)
+ .getAccuracy() : 0;
line++;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/languagemodel/NGramLanguageModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/languagemodel/NGramLanguageModel.java b/opennlp-tools/src/main/java/opennlp/tools/languagemodel/NGramLanguageModel.java
index 4b58c14..ff3039d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/languagemodel/NGramLanguageModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/languagemodel/NGramLanguageModel.java
@@ -122,7 +122,7 @@ public class NGramLanguageModel extends NGramModel implements LanguageModel {
}
private double getLaplaceSmoothingProbability(StringList ngram, StringList nMinusOneToken) {
- return (getCount(ngram) + k) / ((double) getCount(nMinusOneToken) + k * size());
+ return (getCount(ngram) + k) / (getCount(nMinusOneToken) + k * size());
}
private double getStupidBackoffProbability(StringList ngram, StringList nMinusOneToken) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
index 1175ecc..8cc39cc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/BasicContextGenerator.java
@@ -44,7 +44,7 @@ public class BasicContextGenerator implements ContextGenerator {
*/
public String[] getContext(Object o) {
String s = (String) o;
- return (String[]) s.split(separator);
+ return s.split(separator);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java
index f0c1843..3623bd9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/maxent/GISModel.java
@@ -199,7 +199,7 @@ public final class GISModel extends AbstractModel {
prior[oid] = Math
.exp(prior[oid]
* model.getConstantInverse()
- + ((1.0 - ((double) numfeats[oid] / model
+ + ((1.0 - (numfeats[oid] / model
.getCorrectionConstant())) * model.getCorrectionParam()));
} else {
prior[oid] = Math.exp(prior[oid] * model.getConstantInverse());
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
index 438b67a..8bb05d2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/OnePassRealValueDataIndexer.java
@@ -63,7 +63,7 @@ public class OnePassRealValueDataIndexer extends OnePassDataIndexer {
values = new float[numUniqueEvents][];
int numEvents = eventsToCompare.size();
for (int i = 0, j = 0; i < numEvents; i++) {
- ComparableEvent evt = (ComparableEvent) eventsToCompare.get(i);
+ ComparableEvent evt = eventsToCompare.get(i);
if (null == evt) {
continue; // this was a dupe, skip over it.
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
index 044477b..05314ef 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
@@ -315,7 +315,7 @@ public class Parser extends AbstractBottomUpParser {
// TODO: Remove cast for HeadRules
return new ParserModel(languageCode, buildModel, checkModel,
- posModel, chunkModel, (opennlp.tools.parser.HeadRules) rules,
+ posModel, chunkModel, rules,
ParserType.CHUNKING, manifestInfoEntries);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
index c1e54ca..a06576e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/treeinsert/Parser.java
@@ -489,7 +489,7 @@ public class Parser extends AbstractBottomUpParser {
// TODO: Remove cast for HeadRules
return new ParserModel(languageCode, buildModel, checkModel,
attachModel, posModel, chunkModel,
- (opennlp.tools.parser.HeadRules) rules, ParserType.TREEINSERT, manifestInfoEntries);
+ rules, ParserType.TREEINSERT, manifestInfoEntries);
}
public static ParserModel train(String languageCode,
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
index 9602066..ad194ed 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
@@ -166,7 +166,7 @@ public class ChunkerTrainer extends CasConsumer_ImplBase {
chunkIndex.iterator(), containingConstraint);
while (chunkIterator.hasNext()) {
- AnnotationFS chunkAnnotation = (AnnotationFS) chunkIterator.next();
+ AnnotationFS chunkAnnotation = chunkIterator.next();
processChunk(tcas, (chunkAnnotation));
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
index eac01f7..5cf337f 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizer.java
@@ -56,11 +56,11 @@ public class DocumentCategorizer extends AbstractDocumentCategorizer {
protected void setBestCategory(CAS tcas, String bestCategory) {
FSIndex<AnnotationFS> categoryIndex = tcas.getAnnotationIndex(mCategoryType);
- AnnotationFS categoryAnnotation = (AnnotationFS) (categoryIndex.size() > 0 ?
- categoryIndex.iterator().next() : null);
+ AnnotationFS categoryAnnotation = categoryIndex.size() > 0 ?
+ categoryIndex.iterator().next() : null;
if (categoryIndex.size() > 0) {
- categoryAnnotation = (AnnotationFS) categoryIndex.iterator().next();
+ categoryAnnotation = categoryIndex.iterator().next();
}
else {
categoryAnnotation = tcas.createAnnotation(mCategoryType, 0,
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java b/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
index f607332..c0bad5d 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/namefind/AbstractNameFinder.java
@@ -152,17 +152,17 @@ abstract class AbstractNameFinder extends CasAnnotator_ImplBase {
}
Span[] names = find(cas,
- (String[]) sentenceTokenList.toArray(new String[sentenceTokenList.size()]));
+ sentenceTokenList.toArray(new String[sentenceTokenList.size()]));
AnnotationFS nameAnnotations[] = new AnnotationFS[names.length];
for (int i = 0; i < names.length; i++) {
- int startIndex = ((AnnotationFS) sentenceTokenAnnotationList.get(
- names[i].getStart())).getBegin();
+ int startIndex = sentenceTokenAnnotationList.get(
+ names[i].getStart()).getBegin();
- int endIndex = ((AnnotationFS) sentenceTokenAnnotationList.get(
- names[i].getEnd() - 1)).getEnd();
+ int endIndex = sentenceTokenAnnotationList.get(
+ names[i].getEnd() - 1).getEnd();
nameAnnotations[i] =
cas.createAnnotation(mNameType, startIndex, endIndex);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java b/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
index 595c951..e55f4ae 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/normalizer/StringDictionary.java
@@ -57,7 +57,7 @@ public class StringDictionary {
}
public String get(StringList key) {
- return (String) entries.get(key);
+ return entries.get(key);
}
public void put(StringList key, String value) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java b/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
index 3345799..a3b3f00 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/parser/Parser.java
@@ -274,14 +274,13 @@ public class Parser extends CasAnnotator_ImplBase {
List<Span> tokenSpans = new LinkedList<Span>();
while(containingTokens.hasNext()) {
- AnnotationFS token = (AnnotationFS) containingTokens.next();
+ AnnotationFS token = containingTokens.next();
tokenSpans.add(new Span(token.getBegin() - sentenceAnnotation.getBegin(),
token.getEnd() - sentenceAnnotation.getBegin()));
}
- ParseConverter converter = new ParseConverter(sentence,(Span[])
- tokenSpans.toArray(new Span[tokenSpans.size()]));
+ ParseConverter converter = new ParseConverter(sentence,tokenSpans.toArray(new Span[tokenSpans.size()]));
Parse unparsedTree = converter.getParseForTagger();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
index e9bb048..23158b9 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/postag/POSTaggerTrainer.java
@@ -181,7 +181,7 @@ public class POSTaggerTrainer extends CasConsumer_ImplBase {
while (containingTokens.hasNext()) {
- AnnotationFS tokenAnnotation = (AnnotationFS) containingTokens.next();
+ AnnotationFS tokenAnnotation = containingTokens.next();
String tag = tokenAnnotation.getFeatureValueAsString(mPOSFeature);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/d79b97cc/opennlp-uima/src/main/java/opennlp/uima/util/AnnotationComboIterator.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/util/AnnotationComboIterator.java b/opennlp-uima/src/main/java/opennlp/uima/util/AnnotationComboIterator.java
index c7ca4ec..f6cac3f 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/util/AnnotationComboIterator.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/util/AnnotationComboIterator.java
@@ -86,12 +86,12 @@ public class AnnotationComboIterator implements Iterable<AnnotationIteratorPair>
AnnotationComboIterator.this.nextLowerChecked = true;
AnnotationComboIterator.this.nextLowerAvailable = false;
if (AnnotationComboIterator.this.lowerIt.isValid()) {
- AnnotationFS lowerFS = (AnnotationFS) AnnotationComboIterator.this.lowerIt.get();
+ AnnotationFS lowerFS = AnnotationComboIterator.this.lowerIt.get();
int lowerBegin = lowerFS.getBegin();
while (lowerBegin < AnnotationComboIterator.this.upperBegin) {
AnnotationComboIterator.this.lowerIt.moveToNext();
if (AnnotationComboIterator.this.lowerIt.isValid()) {
- lowerFS = (AnnotationFS) AnnotationComboIterator.this.lowerIt.get();
+ lowerFS = AnnotationComboIterator.this.lowerIt.get();
lowerBegin = lowerFS.getBegin();
} else {
return false;
@@ -113,7 +113,7 @@ public class AnnotationComboIterator implements Iterable<AnnotationIteratorPair>
throw new NoSuchElementException();
}
AnnotationComboIterator.this.nextLowerChecked = false;
- final AnnotationFS rv = (AnnotationFS) AnnotationComboIterator.this.lowerIt.get();
+ final AnnotationFS rv = AnnotationComboIterator.this.lowerIt.get();
AnnotationComboIterator.this.lowerIt.moveToNext();
return rv;
}
@@ -160,7 +160,7 @@ public class AnnotationComboIterator implements Iterable<AnnotationIteratorPair>
this.upperIt.moveToFirst();
this.lowerIt.moveToFirst();
if (this.upperIt.isValid()) {
- final AnnotationFS upperFS = (AnnotationFS) this.upperIt.get();
+ final AnnotationFS upperFS = this.upperIt.get();
this.upperBegin = upperFS.getBegin();
this.upperEnd = upperFS.getEnd();
} else {
@@ -176,7 +176,7 @@ public class AnnotationComboIterator implements Iterable<AnnotationIteratorPair>
if (!this.upperIt.hasNext()) {
throw new NoSuchElementException();
}
- final AnnotationFS upperFS = (AnnotationFS) this.upperIt.next();
+ final AnnotationFS upperFS = this.upperIt.next();
this.upperBegin = upperFS.getBegin();
this.upperEnd = upperFS.getEnd();
this.nextLowerChecked = false;
[25/50] [abbrv] opennlp git commit: Remove pointless boolean
expression
Posted by jo...@apache.org.
Remove pointless boolean expression
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/eb93f64b
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/eb93f64b
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/eb93f64b
Branch: refs/heads/889
Commit: eb93f64b9f1150f658ba456b4630e1040dac8112
Parents: 1550dac
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 20:50:22 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:39:07 2016 +0100
----------------------------------------------------------------------
.../src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java | 2 +-
opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/eb93f64b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
index ee785cc..275cf4d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceStream.java
@@ -164,7 +164,7 @@ public class ADSentenceStream extends
/* now we have to take care of the lastLevel. Every time it raises, we will add the
leaf to the node at the top. If it decreases, we remove the top. */
- while (line != null && line.length() != 0 && line.startsWith("</s>") == false && !line.equals("&&")) {
+ while (line != null && line.length() != 0 && !line.startsWith("</s>") && !line.equals("&&")) {
TreeElement element = this.getElement(line);
if(element != null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/eb93f64b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
index d36e89e..1fba758 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSample.java
@@ -245,7 +245,7 @@ public class NameSample {
}
else if (parts[pi].equals(NameSampleDataStream.END_TAG)) {
- if(catchingName == false) {
+ if(!catchingName) {
throw new IOException("Found unexpected annotation: " + errorTokenWithContext(parts, pi));
}
catchingName = false;
[17/50] [abbrv] opennlp git commit: Remove unnecessary parentheses
Posted by jo...@apache.org.
Remove unnecessary parentheses
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/dd483afe
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/dd483afe
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/dd483afe
Branch: refs/heads/889
Commit: dd483afe90b157ab39af6fab1f6f1a5d1775fbe3
Parents: d79b97c
Author: J�rn Kottmann <jo...@apache.org>
Authored: Fri Oct 28 18:54:40 2016 +0200
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:32 2016 +0100
----------------------------------------------------------------------
.../src/main/java/opennlp/tools/chunker/ChunkSample.java | 8 ++++----
.../tools/chunker/DefaultChunkerSequenceValidator.java | 6 +++---
.../java/opennlp/tools/formats/Conll03NameSampleStream.java | 4 ++--
.../java/opennlp/tools/formats/EvalitaNameSampleStream.java | 2 +-
.../opennlp/tools/formats/NameSampleDataStreamFactory.java | 2 +-
.../formats/convert/ParseToTokenSampleStreamFactory.java | 4 ++--
.../src/main/java/opennlp/tools/lemmatizer/LemmaSample.java | 6 +++---
.../src/main/java/opennlp/tools/ml/model/AbstractModel.java | 2 +-
.../main/java/opennlp/tools/ml/model/ComparableEvent.java | 4 ++--
.../java/opennlp/tools/ml/model/ComparablePredicate.java | 4 ++--
.../src/main/java/opennlp/tools/ml/model/MutableContext.java | 2 +-
.../opennlp/tools/ml/model/RealValueFileEventStream.java | 2 +-
.../java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java | 2 +-
.../java/opennlp/tools/namefind/DictionaryNameFinder.java | 2 +-
.../java/opennlp/tools/parser/AbstractContextGenerator.java | 6 +++---
.../java/opennlp/tools/parser/AbstractParserEventStream.java | 2 +-
.../java/opennlp/tools/parser/ChunkContextGenerator.java | 6 +++---
.../opennlp/tools/parser/chunking/BuildContextGenerator.java | 2 +-
.../java/opennlp/tools/sentdetect/SentenceDetectorME.java | 2 +-
.../test/java/opennlp/tools/namefind/NameFinderMETest.java | 2 +-
.../test/java/opennlp/tools/postag/POSTaggerFactoryTest.java | 2 +-
.../src/test/java/opennlp/tools/postag/POSTaggerMETest.java | 2 +-
.../src/main/java/opennlp/uima/chunker/ChunkerTrainer.java | 2 +-
23 files changed, 38 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
index e829f9d..da71d36 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkSample.java
@@ -66,9 +66,9 @@ public class ChunkSample {
validateArguments(sentence.size(), tags.size(), preds.size());
- this.sentence = Collections.unmodifiableList(new ArrayList<String>((sentence)));
- this.tags = Collections.unmodifiableList(new ArrayList<String>((tags)));
- this.preds = Collections.unmodifiableList(new ArrayList<String>((preds)));
+ this.sentence = Collections.unmodifiableList(new ArrayList<String>(sentence));
+ this.tags = Collections.unmodifiableList(new ArrayList<String>(tags));
+ this.preds = Collections.unmodifiableList(new ArrayList<String>(preds));
}
/** Gets the training sentence */
@@ -117,7 +117,7 @@ public class ChunkSample {
for (int ci = 0, cn = aPreds.length; ci < cn; ci++) {
String pred = aPreds[ci];
if (pred.startsWith("B-")
- || (!pred.equals("I-" + startTag) && !pred.equals("O"))) { // start
+ || !pred.equals("I-" + startTag) && !pred.equals("O")) { // start
if (foundPhrase) { // handle the last
phrases.add(new Span(startIndex, ci, startTag));
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/chunker/DefaultChunkerSequenceValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/DefaultChunkerSequenceValidator.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/DefaultChunkerSequenceValidator.java
index 41e0230..ff7136a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/DefaultChunkerSequenceValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/DefaultChunkerSequenceValidator.java
@@ -24,14 +24,14 @@ public class DefaultChunkerSequenceValidator implements SequenceValidator<String
private boolean validOutcome(String outcome, String prevOutcome) {
if (outcome.startsWith("I-")) {
if (prevOutcome == null) {
- return (false);
+ return false;
}
else {
if (prevOutcome.equals("O")) {
- return (false);
+ return false;
}
if (!prevOutcome.substring(2).equals(outcome.substring(2))) {
- return (false);
+ return false;
}
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
index eed8eef..78c0ee8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
@@ -116,12 +116,12 @@ public class Conll03NameSampleStream implements ObjectStream<NameSample>{
String fields[] = line.split(" ");
// For English: WORD POS-TAG SC-TAG NE-TAG
- if (LANGUAGE.EN.equals(lang) && (fields.length == 4)) {
+ if (LANGUAGE.EN.equals(lang) && fields.length == 4) {
sentence.add(fields[0]);
tags.add(fields[3]); // 3 is NE-TAG
}
// For German: WORD LEMA-TAG POS-TAG SC-TAG NE-TAG
- else if (LANGUAGE.DE.equals(lang) && (fields.length == 5)) {
+ else if (LANGUAGE.DE.equals(lang) && fields.length == 5) {
sentence.add(fields[0]);
tags.add(fields[4]); // 4 is NE-TAG
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
index a983b87..026b2a1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
@@ -160,7 +160,7 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample>{
String fields[] = line.split(" ");
// For Italian: WORD POS-TAG SC-TAG NE-TAG
- if (LANGUAGE.IT.equals(lang) && (fields.length == 4)) {
+ if (LANGUAGE.IT.equals(lang) && fields.length == 4) {
sentence.add(fields[0]);
tags.add(fields[3]); // 3 is NE-TAG
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
index 69a5adb..a95c8c6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/NameSampleDataStreamFactory.java
@@ -54,7 +54,7 @@ public class NameSampleDataStreamFactory extends AbstractSampleStreamFactory<Nam
ObjectStream<String> lineStream = null;
try {
- lineStream = new PlainTextByLineStream((sampleDataIn), params.getEncoding());
+ lineStream = new PlainTextByLineStream(sampleDataIn, params.getEncoding());
} catch (IOException ex) {
CmdLineUtil.handleCreateObjectStreamError(ex);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
index 8b13ece..318b562 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/convert/ParseToTokenSampleStreamFactory.java
@@ -46,8 +46,8 @@ public class ParseToTokenSampleStreamFactory extends DetokenizerSampleStreamFact
StreamFactoryRegistry.DEFAULT_FORMAT).create(
ArgumentParser.filter(args, WordTagSampleStreamFactory.Parameters.class));
- return (new POSToTokenSampleStream(createDetokenizer(params),
- new ParseToPOSSampleStream(parseSampleStream)));
+ return new POSToTokenSampleStream(createDetokenizer(params),
+ new ParseToPOSSampleStream(parseSampleStream));
}
public static void registerFactory() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
index a743575..fe6dd69 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/LemmaSample.java
@@ -58,9 +58,9 @@ public LemmaSample(String[] tokens, String[] tags, String[] lemmas) {
validateArguments(tokens.size(), tags.size(), lemmas.size());
- this.tokens = Collections.unmodifiableList(new ArrayList<String>((tokens)));
- this.tags = Collections.unmodifiableList(new ArrayList<String>((tags)));
- this.lemmas = Collections.unmodifiableList(new ArrayList<String>((lemmas)));
+ this.tokens = Collections.unmodifiableList(new ArrayList<String>(tokens));
+ this.tags = Collections.unmodifiableList(new ArrayList<String>(tags));
+ this.lemmas = Collections.unmodifiableList(new ArrayList<String>(lemmas));
}
public String[] getTokens() {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
index 8ce2e3a..0ab1fc6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/AbstractModel.java
@@ -139,7 +139,7 @@ public abstract class AbstractModel implements MaxentModel {
}
public int getNumOutcomes() {
- return(evalParams.getNumOutcomes());
+ return evalParams.getNumOutcomes();
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparableEvent.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparableEvent.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparableEvent.java
index a1d9e78..83ed078 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparableEvent.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparableEvent.java
@@ -49,8 +49,8 @@ public class ComparableEvent implements Comparable<ComparableEvent> {
else if (outcome > ce.outcome)
return 1;
- int smallerLength = (predIndexes.length > ce.predIndexes.length ? ce.predIndexes.length
- : predIndexes.length);
+ int smallerLength = predIndexes.length > ce.predIndexes.length ? ce.predIndexes.length
+ : predIndexes.length;
for (int i = 0; i < smallerLength; i++) {
if (predIndexes[i] < ce.predIndexes[i])
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparablePredicate.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparablePredicate.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparablePredicate.java
index 561f13b..7483a26 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparablePredicate.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/ComparablePredicate.java
@@ -36,8 +36,8 @@ public class ComparablePredicate implements Comparable<ComparablePredicate> {
}
public int compareTo(ComparablePredicate cp) {
- int smallerLength = (outcomes.length > cp.outcomes.length?
- cp.outcomes.length : outcomes.length);
+ int smallerLength = outcomes.length > cp.outcomes.length?
+ cp.outcomes.length : outcomes.length;
for (int i=0; i<smallerLength; i++) {
if (outcomes[i] < cp.outcomes[i]) return -1;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/ml/model/MutableContext.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/MutableContext.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/MutableContext.java
index 3685974..36e2cf1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/MutableContext.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/MutableContext.java
@@ -59,6 +59,6 @@ public class MutableContext extends Context {
}
public boolean contains(int outcome) {
- return(Arrays.binarySearch(outcomes,outcome) >= 0);
+ return Arrays.binarySearch(outcomes,outcome) >= 0;
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java
index 68b3503..1c9f430 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/model/RealValueFileEventStream.java
@@ -86,7 +86,7 @@ public class RealValueFileEventStream extends FileEventStream {
String outcome = line.substring(0, si);
String[] contexts = line.substring(si + 1).split(" ");
float[] values = parseContexts(contexts);
- return (new Event(outcome, contexts, values));
+ return new Event(outcome, contexts, values);
}
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
index 23ec75a..05503fa 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ml/naivebayes/NaiveBayesModel.java
@@ -136,7 +136,7 @@ public class NaiveBayesModel extends AbstractModel {
else if (denominator == 0 || denominator < Double.MIN_VALUE)
return 0;
else
- return 1.0 * (numerator) / (denominator);
+ return 1.0 * numerator / denominator;
}
private static double getSmoothedProbability(double numerator, double denominator, double vocabulary) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
index 746aeb0..b9e1f17 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/DictionaryNameFinder.java
@@ -90,7 +90,7 @@ public class DictionaryNameFinder implements TokenNameFinder {
if (nameFound != null) {
namesFound.add(nameFound);
// skip over the found tokens for the next search
- offsetFrom += (nameFound.length() - 1);
+ offsetFrom += nameFound.length() - 1;
}
}
return namesFound.toArray(new Span[namesFound.size()]);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
index 5cfff7d..f537576 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
@@ -58,7 +58,7 @@ public abstract class AbstractContextGenerator {
StringBuilder feat = new StringBuilder(5);
feat.append(i).append("=");
feat.append(punct.getType());
- return (feat.toString());
+ return feat.toString();
}
protected String cons(Parse p, int i) {
@@ -73,7 +73,7 @@ public abstract class AbstractContextGenerator {
else {
feat.append(EOS);
}
- return (feat.toString());
+ return feat.toString();
}
protected String consbo(Parse p, int i) { //cons back-off
@@ -88,7 +88,7 @@ public abstract class AbstractContextGenerator {
else {
feat.append(EOS);
}
- return (feat.toString());
+ return feat.toString();
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
index 8913c3c..9d72bf8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
@@ -199,7 +199,7 @@ public abstract class AbstractParserEventStream extends opennlp.tools.util.Abstr
*/
protected boolean lastChild(Parse child, Parse parent) {
Parse[] kids = AbstractBottomUpParser.collapsePunctuation(parent.getChildren(),punctSet);
- return (kids[kids.length - 1] == child);
+ return kids[kids.length - 1] == child;
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
index 7471b3c..d3d8303 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkContextGenerator.java
@@ -164,7 +164,7 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
if (contextsCache != null) {
contextsCache.put(cacheKey,contexts);
}
- return (contexts);
+ return contexts;
}
private String chunkandpostag(int i, String tok, String tag, String chunk) {
@@ -173,7 +173,7 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
if (i < 0) {
feat.append("|").append(chunk);
}
- return (feat.toString());
+ return feat.toString();
}
private String chunkandpostagbo(int i, String tag, String chunk) {
@@ -182,6 +182,6 @@ public class ChunkContextGenerator implements ChunkerContextGenerator {
if (i < 0) {
feat.append("|").append(chunk);
}
- return (feat.toString());
+ return feat.toString();
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
index 01e9a44..081ef9a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
@@ -304,6 +304,6 @@ public class BuildContextGenerator extends AbstractContextGenerator {
}
}
}
- return (features.toArray(new String[features.size()]));
+ return features.toArray(new String[features.size()]);
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
index fce4e52..4998545 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorME.java
@@ -215,7 +215,7 @@ public class SentenceDetectorME implements SentenceDetector {
while (end > 0 && StringUtil.isWhitespace(s.charAt(end - 1)))
end--;
- if ((end - start) > 0) {
+ if (end - start > 0) {
sentProbs.add(1d);
return new Span[] {new Span(start, end)};
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java b/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
index 2fe40d1..cc8b0d1 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/namefind/NameFinderMETest.java
@@ -272,7 +272,7 @@ public class NameFinderMETest {
// now test if it can detect the sample sentences
- String[] sentence = ("NATO United States Barack Obama").split("\\s+");
+ String[] sentence = "NATO United States Barack Obama".split("\\s+");
Span[] names1 = nameFinder.find(sentence);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
index fbab448..2f72124 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerFactoryTest.java
@@ -134,4 +134,4 @@ public class POSTaggerFactoryTest {
public void testCreateWithHierarchy2() throws InvalidFormatException {
POSTaggerFactory.create(this.getClass().getCanonicalName(), null, null);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
index 996b233..bcac9ae 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
@@ -89,4 +89,4 @@ public class POSTaggerMETest {
POSTaggerME.buildNGramDictionary(samples, 0);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/dd483afe/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
index ad194ed..d22879d 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/chunker/ChunkerTrainer.java
@@ -167,7 +167,7 @@ public class ChunkerTrainer extends CasConsumer_ImplBase {
while (chunkIterator.hasNext()) {
AnnotationFS chunkAnnotation = chunkIterator.next();
- processChunk(tcas, (chunkAnnotation));
+ processChunk(tcas, chunkAnnotation);
}
}
[35/50] [abbrv] opennlp git commit: Remove deprecated code from
PlainTextByLineStream
Posted by jo...@apache.org.
Remove deprecated code from PlainTextByLineStream
See issue OPENNLP-882
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/727964d7
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/727964d7
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/727964d7
Branch: refs/heads/889
Commit: 727964d7efad28d6b61d6d0597386415b0ab7075
Parents: 554626d
Author: William Colen <co...@apache.org>
Authored: Tue Dec 20 22:44:16 2016 -0200
Committer: William Colen <co...@apache.org>
Committed: Tue Dec 20 22:44:16 2016 -0200
----------------------------------------------------------------------
.../formats/BioNLP2004NameSampleStream.java | 14 -----
.../BioNLP2004NameSampleStreamFactory.java | 10 +++-
.../tools/formats/Conll02NameSampleStream.java | 19 -------
.../tools/formats/Conll03NameSampleStream.java | 26 ++-------
.../tools/formats/EvalitaNameSampleStream.java | 20 -------
.../formats/NameFinderCensus90NameStream.java | 17 ------
.../tools/formats/ad/ADChunkSampleStream.java | 21 --------
.../tools/formats/ad/ADNameSampleStream.java | 32 +----------
.../tools/formats/ad/ADPOSSampleStream.java | 29 ----------
.../formats/ad/ADSentenceSampleStream.java | 25 ---------
.../namefind/TokenNameFinderEvaluator.java | 56 --------------------
.../tools/util/PlainTextByLineStream.java | 47 ----------------
12 files changed, 12 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
index 6825d80..14eb42e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStream.java
@@ -18,7 +18,6 @@
package opennlp.tools.formats;
import java.io.IOException;
-import java.io.InputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
@@ -67,19 +66,6 @@ public class BioNLP2004NameSampleStream implements ObjectStream<NameSample> {
}
- @Deprecated
- public BioNLP2004NameSampleStream(InputStream in, int types) {
- try {
- this.lineStream = new PlainTextByLineStream(in, "UTF-8");
- System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
-
- this.types = types;
- }
-
public NameSample read() throws IOException {
List<String> sentence = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
index 3920a20..6445a4d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactory.java
@@ -17,6 +17,8 @@
package opennlp.tools.formats;
+import java.io.IOException;
+
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
import opennlp.tools.cmdline.CmdLineUtil;
@@ -68,7 +70,11 @@ public class BioNLP2004NameSampleStreamFactory extends AbstractSampleStreamFacto
BioNLP2004NameSampleStream.GENERATE_RNA_ENTITIES;
}
- return new BioNLP2004NameSampleStream(
- CmdLineUtil.openInFile(params.getData()), typesToGenerate);
+ try {
+ return new BioNLP2004NameSampleStream(
+ CmdLineUtil.createInputStreamFactory(params.getData()), typesToGenerate);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
index 7cd3810..efb37a0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
@@ -18,7 +18,6 @@
package opennlp.tools.formats;
import java.io.IOException;
-import java.io.InputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
@@ -84,24 +83,6 @@ public class Conll02NameSampleStream implements ObjectStream<NameSample>{
this.types = types;
}
- /**
- * @param lang the language of the CONLL 02 data
- * @param in an Input Stream to read data.
- * @param types the entity types to include in the Name Samples
- */
- @Deprecated
- public Conll02NameSampleStream(LANGUAGE lang, InputStream in, int types) {
- this.lang = lang;
- try {
- this.lineStream = new PlainTextByLineStream(in, "UTF-8");
- System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- this.types = types;
- }
-
static Span extract(int begin, int end, String beginTag) throws InvalidFormatException {
String type = beginTag.substring(2);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
index 07b62e8..a9e2e64 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll03NameSampleStream.java
@@ -15,12 +15,14 @@
package opennlp.tools.formats;
+import static opennlp.tools.formats.Conll02NameSampleStream.extract;
+
import java.io.IOException;
-import java.io.InputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
+
import opennlp.tools.namefind.NameSample;
import opennlp.tools.util.InputStreamFactory;
import opennlp.tools.util.ObjectStream;
@@ -28,8 +30,6 @@ import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.StringUtil;
-import static opennlp.tools.formats.Conll02NameSampleStream.extract;
-
/**
* An import stream which can parse the CONLL03 data.
*/
@@ -70,26 +70,6 @@ public class Conll03NameSampleStream implements ObjectStream<NameSample>{
this.types = types;
}
- /**
- *
- * @param lang the language of the CONLL 03 data
- * @param in the Input Stream to read the data file
- * @param types the entity types to include in the Name Sample object stream
- */
- @Deprecated
- public Conll03NameSampleStream(LANGUAGE lang, InputStream in, int types) {
-
- this.lang = lang;
- try {
- this.lineStream = new PlainTextByLineStream(in, "UTF-8");
- System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- this.types = types;
- }
-
public NameSample read() throws IOException {
List<String> sentence = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
index 026b2a1..925a130 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/EvalitaNameSampleStream.java
@@ -18,7 +18,6 @@
package opennlp.tools.formats;
import java.io.IOException;
-import java.io.InputStream;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
@@ -92,25 +91,6 @@ public class EvalitaNameSampleStream implements ObjectStream<NameSample>{
this.types = types;
}
- /**
- * @param lang the language of the Evalita data file
- * @param in an Input Stream to read data.
- * @param types the types of the entities which are included in the Name Sample stream
- */
- @Deprecated
- public EvalitaNameSampleStream(LANGUAGE lang, InputStream in, int types) {
-
- this.lang = lang;
- try {
- this.lineStream = new PlainTextByLineStream(in, "UTF-8");
- System.setOut(new PrintStream(System.out, true, "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- this.types = types;
- }
-
static final Span extract(int begin, int end, String beginTag) throws InvalidFormatException {
String type = beginTag.substring(2);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java
index ee3f933..7773e50 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/NameFinderCensus90NameStream.java
@@ -16,7 +16,6 @@
package opennlp.tools.formats;
import java.io.IOException;
-import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.Locale;
@@ -76,22 +75,6 @@ public class NameFinderCensus90NameStream implements ObjectStream<StringList> {
this.lineStream = new PlainTextByLineStream(in, this.encoding);
}
-
- /**
- * This constructor takes an <code>InputStream</code> and a <code>Charset</code>
- * and opens an associated stream object with the specified encoding specified.
- *
- * @param in an <code>InputStream</code> for the input file.
- * @param encoding the <code>Charset</code> to apply to the input stream.
- *
- * @deprecated use {@link NameFinderCensus90NameStream#NameFinderCensus90NameStream(InputStreamFactory, Charset)}
- */
- public NameFinderCensus90NameStream(InputStream in, Charset encoding) {
- this.locale = new Locale("en"); // locale is English
- this.encoding = encoding;
- this.lineStream = new PlainTextByLineStream(in, this.encoding);
- }
-
public StringList read() throws IOException {
String line = lineStream.read();
StringList name = null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
index d176f8b..12faf4c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADChunkSampleStream.java
@@ -18,7 +18,6 @@
package opennlp.tools.formats.ad;
import java.io.IOException;
-import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
@@ -91,26 +90,6 @@ public class ADChunkSampleStream implements ObjectStream<ChunkSample> {
}
}
- /**
- * Creates a new {@link NameSample} stream from a {@link InputStream}
- *
- * @param in
- * the Corpus {@link InputStream}
- * @param charsetName
- * the charset of the Arvores Deitadas Corpus
- */
- @Deprecated
- public ADChunkSampleStream(InputStream in, String charsetName) {
-
- try {
- this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- in, charsetName));
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- }
-
public ChunkSample read() throws IOException {
Sentence paragraph;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
index abf6dab..97b6ab9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java
@@ -201,31 +201,6 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
}
}
- /**
- * Creates a new {@link NameSample} stream from a {@link InputStream}
- *
- * @param in
- * the Corpus {@link InputStream}
- * @param charsetName
- * the charset of the Arvores Deitadas Corpus
- * @param splitHyphenatedTokens
- * if true hyphenated tokens will be separated: "carros-monstro" >
- * "carros" "-" "monstro"
- */
- @Deprecated
- public ADNameSampleStream(InputStream in, String charsetName,
- boolean splitHyphenatedTokens) {
-
- try {
- this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- in, charsetName));
- this.splitHyphenatedTokens = splitHyphenatedTokens;
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- }
-
private int textID = -1;
public NameSample read() throws IOException {
@@ -347,17 +322,12 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
// a NER.
// we check if it is true, and expand the last NER
int lastIndex = names.size() - 1;
- boolean error = false;
if (names.size() > 0) {
Span last = names.get(lastIndex);
if (last.getEnd() == sentence.size() - 1) {
names.set(lastIndex, new Span(last.getStart(), sentence.size(),
last.getType()));
- } else {
- error = true;
- }
- } else {
- error = true;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
index ff57d83..3f8fdb7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADPOSSampleStream.java
@@ -91,35 +91,6 @@ public class ADPOSSampleStream implements ObjectStream<POSSample> {
}
}
- /**
- * Creates a new {@link POSSample} stream from a {@link InputStream}
- *
- * @param in
- * the Corpus {@link InputStream}
- * @param charsetName
- * the charset of the Arvores Deitadas Corpus
- * @param expandME
- * if true will expand the multiword expressions, each word of the
- * expression will have the POS Tag that was attributed to the
- * expression plus the prefix B- or I- (CONLL convention)
- * @param includeFeatures
- * if true will combine the POS Tag with the feature tags
- */
- @Deprecated
- public ADPOSSampleStream(InputStream in, String charsetName,
- boolean expandME, boolean includeFeatures) {
-
- try {
- this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- in, charsetName));
- this.expandME = expandME;
- this.isIncludeFeatures = includeFeatures;
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- }
-
public POSSample read() throws IOException {
Sentence paragraph;
while ((paragraph = this.adSentenceStream.read()) != null) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
index d3a04df..db09bae 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/ad/ADSentenceSampleStream.java
@@ -92,31 +92,6 @@ public class ADSentenceSampleStream implements ObjectStream<SentenceSample> {
this.isIncludeTitles = includeHeadlines;
}
- /**
- * Creates a new {@link SentenceSample} stream from a {@link FileInputStream}
- *
- * @param in
- * input stream from the corpus
- * @param charsetName
- * the charset to use while reading the corpus
- * @param includeHeadlines
- * if true will output the sentences marked as news headlines
- */
- @Deprecated
- public ADSentenceSampleStream(FileInputStream in, String charsetName,
- boolean includeHeadlines) {
- try {
- this.adSentenceStream = new ADSentenceStream(new PlainTextByLineStream(
- in, charsetName));
- } catch (UnsupportedEncodingException e) {
- // UTF-8 is available on all JVMs, will never happen
- throw new IllegalStateException(e);
- }
- ptEosCharacters = Factory.ptEosCharacters;
- Arrays.sort(ptEosCharacters);
- this.isIncludeTitles = includeHeadlines;
- }
-
// The Arvores Deitadas Corpus has information about texts and paragraphs.
public SentenceSample read() throws IOException {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
index fb689ab..3c8c549 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
@@ -18,13 +18,6 @@
package opennlp.tools.namefind;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import opennlp.tools.cmdline.PerformanceMonitor;
-import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.FMeasure;
@@ -99,53 +92,4 @@ public class TokenNameFinderEvaluator extends Evaluator<NameSample> {
public FMeasure getFMeasure() {
return fmeasure;
}
-
- @Deprecated
- public static void main(String[] args) throws IOException {
-
- if (args.length == 4) {
-
- System.out.println("Loading name finder model ...");
- InputStream modelIn = new FileInputStream(args[3]);
-
- TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
-
- TokenNameFinder nameFinder = new NameFinderME(model);
-
- System.out.println("Performing evaluation ...");
- TokenNameFinderEvaluator evaluator = new TokenNameFinderEvaluator(nameFinder);
-
- final NameSampleDataStream sampleStream = new NameSampleDataStream(
- new PlainTextByLineStream(new InputStreamReader(new FileInputStream(args[2]), args[1])));
-
- final PerformanceMonitor monitor = new PerformanceMonitor("sent");
-
- monitor.startAndPrintThroughput();
-
- ObjectStream<NameSample> iterator = new ObjectStream<NameSample>() {
-
- public NameSample read() throws IOException {
- monitor.incrementCounter();
- return sampleStream.read();
- }
-
- public void reset() throws IOException {
- sampleStream.reset();
- }
-
- public void close() throws IOException {
- sampleStream.close();
- }
- };
-
- evaluator.evaluate(iterator);
-
- monitor.stopAndPrintFinalResult();
-
- System.out.println();
- System.out.println("F-Measure: " + evaluator.getFMeasure().getFMeasure());
- System.out.println("Recall: " + evaluator.getFMeasure().getRecallScore());
- System.out.println("Precision: " + evaluator.getFMeasure().getPrecisionScore());
- }
- }
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/727964d7/opennlp-tools/src/main/java/opennlp/tools/util/PlainTextByLineStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/PlainTextByLineStream.java b/opennlp-tools/src/main/java/opennlp/tools/util/PlainTextByLineStream.java
index 2a3fd66..9c7cdc1 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/PlainTextByLineStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/PlainTextByLineStream.java
@@ -20,10 +20,7 @@ package opennlp.tools.util;
import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InputStreamReader;
-import java.io.Reader;
-import java.io.UnsupportedEncodingException;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.Charset;
@@ -52,50 +49,6 @@ public class PlainTextByLineStream implements ObjectStream<String> {
reset();
}
- /**
- * Initializes the current instance.
- *
- * @param in
- * @deprecated Use {@link #PlainTextByLineStream(InputStreamFactory, Charset)} instead.
- */
- public PlainTextByLineStream(Reader in) {
- this.in = new BufferedReader(in);
- this.channel = null;
- this.encoding = null;
- }
-
- /**
- * @deprecated Use {@link #PlainTextByLineStream(InputStreamFactory, String)} instead.
- */
- public PlainTextByLineStream(InputStream in, String charsetName) throws UnsupportedEncodingException {
- this(new InputStreamReader(in, charsetName));
- }
-
- /**
- * @deprecated Use {@link #PlainTextByLineStream(InputStreamFactory, Charset)} instead.
- */
- public PlainTextByLineStream(InputStream in, Charset charset) {
- this(new InputStreamReader(in, charset));
- }
-
- /**
- * @deprecated Use {@link #PlainTextByLineStream(InputStreamFactory, String)} instead.
- */
- public PlainTextByLineStream(FileChannel channel, String charsetName) {
- this.encoding = charsetName;
- this.channel = channel;
-
- // TODO: Why isn't reset called here ?
- in = new BufferedReader(Channels.newReader(channel, encoding));
- }
-
- /**
- * @deprecated Use {@link #PlainTextByLineStream(InputStreamFactory, Charset)} instead.
- */
- public PlainTextByLineStream(FileChannel channel, Charset encoding) {
- this(channel, encoding.name());
- }
-
public String read() throws IOException {
return in.readLine();
}
[37/50] [abbrv] opennlp git commit: Increase backward compatibility
to two minor versions
Posted by jo...@apache.org.
Increase backward compatibility to two minor versions
See issue OPENNLP-896
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/2b0658c5
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/2b0658c5
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/2b0658c5
Branch: refs/heads/889
Commit: 2b0658c59d6285547a5c6de7ac71c7cb134ce975
Parents: 0debe9c
Author: Joern Kottmann <jo...@apache.org>
Authored: Wed Dec 21 11:28:40 2016 +0100
Committer: Joern Kottmann <jo...@apache.org>
Committed: Wed Dec 21 11:36:28 2016 +0100
----------------------------------------------------------------------
.../src/main/java/opennlp/tools/util/model/BaseModel.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/2b0658c5/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
index 6e84715..fdae63a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
@@ -422,7 +422,8 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
if (Version.currentVersion().getMajor() != version.getMajor() ||
Version.currentVersion().getMinor() != version.getMinor()) {
//this check allows for the use of models one minor release behind current minor release
- if(Version.currentVersion().getMajor() == version.getMajor() && (Version.currentVersion().getMinor()-1) != version.getMinor()){
+ if (Version.currentVersion().getMajor() == version.getMajor() && (Version.currentVersion().getMinor() - 2)
+ != version.getMinor()) {
throw new InvalidFormatException("Model version " + version + " is not supported by this ("
+ Version.currentVersion() +") version of OpenNLP!");
}
[43/50] [abbrv] opennlp git commit: Define update and
clearAdaptiveData as default methods
Posted by jo...@apache.org.
Define update and clearAdaptiveData as default methods
See issue OPENNLP-856
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/99323ad6
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/99323ad6
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/99323ad6
Branch: refs/heads/889
Commit: 99323ad68c5eb3f9bae4314984931acca3699521
Parents: 71e40d0
Author: Kottmann <jo...@apache.org>
Authored: Thu Dec 22 17:31:54 2016 +0100
Committer: Kottmann <jo...@apache.org>
Committed: Thu Dec 22 17:35:52 2016 +0100
----------------------------------------------------------------------
.../opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java | 4 ++--
.../tools/util/featuregen/AdditionalContextFeatureGenerator.java | 2 +-
.../tools/util/featuregen/BigramNameFeatureGenerator.java | 2 +-
.../tools/util/featuregen/BrownBigramFeatureGenerator.java | 2 +-
.../tools/util/featuregen/BrownTokenClassFeatureGenerator.java | 2 +-
.../tools/util/featuregen/BrownTokenFeatureGenerator.java | 2 +-
.../tools/util/featuregen/CharacterNgramFeatureGenerator.java | 2 +-
.../tools/util/featuregen/DictionaryFeatureGenerator.java | 2 +-
.../tools/util/featuregen/DocumentBeginFeatureGenerator.java | 2 +-
.../tools/util/featuregen/FastTokenClassFeatureGenerator.java | 2 +-
.../opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java | 3 +++
.../main/java/opennlp/tools/util/featuregen/InSpanGenerator.java | 2 +-
.../tools/util/featuregen/OutcomePriorFeatureGenerator.java | 2 +-
.../opennlp/tools/util/featuregen/PrefixFeatureGenerator.java | 2 +-
.../opennlp/tools/util/featuregen/SentenceFeatureGenerator.java | 2 +-
.../opennlp/tools/util/featuregen/SuffixFeatureGenerator.java | 2 +-
.../tools/util/featuregen/TokenClassFeatureGenerator.java | 2 +-
.../opennlp/tools/util/featuregen/TokenFeatureGenerator.java | 2 +-
.../tools/util/featuregen/TokenPatternFeatureGenerator.java | 2 +-
.../tools/util/featuregen/TrigramNameFeatureGenerator.java | 2 +-
.../tools/util/featuregen/WordClusterFeatureGenerator.java | 2 +-
.../opennlp/tools/util/featuregen/IdentityFeatureGenerator.java | 2 +-
22 files changed, 25 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
index 6969504..ce8e9e4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdaptiveFeatureGenerator.java
@@ -58,11 +58,11 @@ public interface AdaptiveFeatureGenerator {
* @param tokens The tokens of the sentence or other text unit which has been processed.
* @param outcomes The outcomes associated with the specified tokens.
*/
- void updateAdaptiveData(String[] tokens, String[] outcomes);
+ default void updateAdaptiveData(String[] tokens, String[] outcomes) {};
/**
* Informs the feature generator that the context of the adaptive data (typically a document)
* is no longer valid.
*/
- void clearAdaptiveData();
+ default void clearAdaptiveData() {};
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdditionalContextFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdditionalContextFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdditionalContextFeatureGenerator.java
index 233c31d..3606bfc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdditionalContextFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/AdditionalContextFeatureGenerator.java
@@ -24,7 +24,7 @@ import java.util.List;
* The {@link AdditionalContextFeatureGenerator} generates the context from the passed
* in additional context.
*/
-public class AdditionalContextFeatureGenerator extends FeatureGeneratorAdapter {
+public class AdditionalContextFeatureGenerator implements AdaptiveFeatureGenerator {
private String[][] additionalContext;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BigramNameFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BigramNameFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BigramNameFeatureGenerator.java
index a3d16d8..779628b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BigramNameFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BigramNameFeatureGenerator.java
@@ -19,7 +19,7 @@ package opennlp.tools.util.featuregen;
import java.util.List;
-public class BigramNameFeatureGenerator extends FeatureGeneratorAdapter {
+public class BigramNameFeatureGenerator implements AdaptiveFeatureGenerator {
public void createFeatures(List<String> features, String[] tokens, int index, String[] previousOutcomes) {
String wc = FeatureGeneratorUtil.tokenFeature(tokens[index]);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownBigramFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownBigramFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownBigramFeatureGenerator.java
index 9329757..a932e74 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownBigramFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownBigramFeatureGenerator.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
* Generates Brown cluster features for token bigrams.
*/
-public class BrownBigramFeatureGenerator extends FeatureGeneratorAdapter {
+public class BrownBigramFeatureGenerator implements AdaptiveFeatureGenerator {
private BrownCluster brownLexicon;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenClassFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenClassFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenClassFeatureGenerator.java
index 8f53be9..db3d774 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenClassFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenClassFeatureGenerator.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
* Generates Brown cluster features for current token and token class.
*/
-public class BrownTokenClassFeatureGenerator extends FeatureGeneratorAdapter {
+public class BrownTokenClassFeatureGenerator implements AdaptiveFeatureGenerator {
private BrownCluster brownLexicon;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenFeatureGenerator.java
index 5689897..f41203f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/BrownTokenFeatureGenerator.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
* Generates Brown cluster features for current token.
*/
-public class BrownTokenFeatureGenerator extends FeatureGeneratorAdapter {
+public class BrownTokenFeatureGenerator implements AdaptiveFeatureGenerator {
private BrownCluster brownLexicon;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CharacterNgramFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CharacterNgramFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CharacterNgramFeatureGenerator.java
index 6314dac..8c0edd9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CharacterNgramFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/CharacterNgramFeatureGenerator.java
@@ -28,7 +28,7 @@ import opennlp.tools.util.StringUtil;
* generate features about each token.
* The minimum and maximum length can be specified.
*/
-public class CharacterNgramFeatureGenerator extends FeatureGeneratorAdapter {
+public class CharacterNgramFeatureGenerator implements AdaptiveFeatureGenerator {
private final int minLength;
private final int maxLength;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DictionaryFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DictionaryFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DictionaryFeatureGenerator.java
index bbedcc2..68037ed 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DictionaryFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DictionaryFeatureGenerator.java
@@ -31,7 +31,7 @@ import opennlp.tools.namefind.DictionaryNameFinder;
* @see DictionaryNameFinder
* @see InSpanGenerator
*/
-public class DictionaryFeatureGenerator extends FeatureGeneratorAdapter {
+public class DictionaryFeatureGenerator implements AdaptiveFeatureGenerator {
private InSpanGenerator isg;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
index b201e9a..efcfce4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGenerator.java
@@ -19,7 +19,7 @@ package opennlp.tools.util.featuregen;
import java.util.List;
-public class DocumentBeginFeatureGenerator extends FeatureGeneratorAdapter {
+public class DocumentBeginFeatureGenerator implements AdaptiveFeatureGenerator {
private String firstSentence[];
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
index 980d2ad..5c82733 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FastTokenClassFeatureGenerator.java
@@ -30,7 +30,7 @@ import opennlp.tools.util.StringUtil;
* @deprecated Use {@link TokenClassFeatureGenerator} instead!
*/
@Deprecated
-public class FastTokenClassFeatureGenerator extends FeatureGeneratorAdapter {
+public class FastTokenClassFeatureGenerator implements AdaptiveFeatureGenerator {
private static final String TOKEN_CLASS_PREFIX = "wc";
private static final String TOKEN_AND_CLASS_PREFIX = "w&c";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
index f2947cf..8f444ce 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/FeatureGeneratorAdapter.java
@@ -22,7 +22,10 @@ package opennlp.tools.util.featuregen;
* This class provides empty implementations of some of the optional methods in
* {@link AdditionalContextFeatureGenerator} to make implementing feature generators
* easier.
+ *
+ * @deprecated use AdaptiveFeatureGenerator instead
*/
+@Deprecated // in 1.7.0
public abstract class FeatureGeneratorAdapter implements AdaptiveFeatureGenerator {
public void updateAdaptiveData(String[] tokens, String[] outcomes) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java
index acd81ba..bcffc7e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/InSpanGenerator.java
@@ -27,7 +27,7 @@ import opennlp.tools.util.Span;
* Generates features if the tokens are recognized by the provided
* {@link TokenNameFinder}.
*/
-public class InSpanGenerator extends FeatureGeneratorAdapter {
+public class InSpanGenerator implements AdaptiveFeatureGenerator {
private final String prefix;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/OutcomePriorFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/OutcomePriorFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/OutcomePriorFeatureGenerator.java
index 9593e8d..c2281f5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/OutcomePriorFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/OutcomePriorFeatureGenerator.java
@@ -23,7 +23,7 @@ import java.util.List;
/**
* The definition feature maps the underlying distribution of outcomes.
*/
-public class OutcomePriorFeatureGenerator extends FeatureGeneratorAdapter {
+public class OutcomePriorFeatureGenerator implements AdaptiveFeatureGenerator {
public static final String OUTCOME_PRIOR_FEATURE = "def";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
index 10b9c55..8d433b6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/PrefixFeatureGenerator.java
@@ -19,7 +19,7 @@ package opennlp.tools.util.featuregen;
import java.util.List;
-public class PrefixFeatureGenerator extends FeatureGeneratorAdapter {
+public class PrefixFeatureGenerator implements AdaptiveFeatureGenerator {
private static final int PREFIX_LENGTH = 4;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java
index 66b10d4..f7b81dc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SentenceFeatureGenerator.java
@@ -22,7 +22,7 @@ import java.util.List;
/**
* This feature generator creates sentence begin and end features.
*/
-public class SentenceFeatureGenerator extends FeatureGeneratorAdapter {
+public class SentenceFeatureGenerator implements AdaptiveFeatureGenerator {
private final boolean isGenerateFirstWordFeature;
private final boolean isGenerateLastWordFeature;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java
index 5ec34aa..815cc9e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/SuffixFeatureGenerator.java
@@ -19,7 +19,7 @@ package opennlp.tools.util.featuregen;
import java.util.List;
-public class SuffixFeatureGenerator extends FeatureGeneratorAdapter {
+public class SuffixFeatureGenerator implements AdaptiveFeatureGenerator {
private static final int SUFFIX_LENGTH = 4;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
index 4aad40c..7358d24 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenClassFeatureGenerator.java
@@ -25,7 +25,7 @@ import opennlp.tools.util.StringUtil;
/**
* Generates features for different for the class of the token.
*/
-public class TokenClassFeatureGenerator extends FeatureGeneratorAdapter {
+public class TokenClassFeatureGenerator implements AdaptiveFeatureGenerator {
private static final String TOKEN_CLASS_PREFIX = "wc";
private static final String TOKEN_AND_CLASS_PREFIX = "w&c";
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenFeatureGenerator.java
index b0c9c5a..5aee49b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenFeatureGenerator.java
@@ -25,7 +25,7 @@ import opennlp.tools.util.StringUtil;
/**
* Generates a feature which contains the token itself.
*/
-public class TokenFeatureGenerator extends FeatureGeneratorAdapter {
+public class TokenFeatureGenerator implements AdaptiveFeatureGenerator {
private static final String WORD_PREFIX = "w";
private boolean lowercase;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
index 48a855f..79e61b2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TokenPatternFeatureGenerator.java
@@ -29,7 +29,7 @@ import opennlp.tools.util.StringUtil;
* Partitions tokens into sub-tokens based on character classes and generates
* class features for each of the sub-tokens and combinations of those sub-tokens.
*/
-public class TokenPatternFeatureGenerator extends FeatureGeneratorAdapter {
+public class TokenPatternFeatureGenerator implements AdaptiveFeatureGenerator {
private Pattern noLetters = Pattern.compile("[^a-zA-Z]");
private Tokenizer tokenizer;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TrigramNameFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TrigramNameFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TrigramNameFeatureGenerator.java
index 0fa61e0..7724eb5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TrigramNameFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/TrigramNameFeatureGenerator.java
@@ -23,7 +23,7 @@ import java.util.List;
* Adds trigram features based on tokens and token classes.
*
*/
-public class TrigramNameFeatureGenerator extends FeatureGeneratorAdapter {
+public class TrigramNameFeatureGenerator implements AdaptiveFeatureGenerator {
public void createFeatures(List<String> features, String[] tokens, int index,
String[] previousOutcomes) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WordClusterFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WordClusterFeatureGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WordClusterFeatureGenerator.java
index 4bd0a6d..11e0e6b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WordClusterFeatureGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/WordClusterFeatureGenerator.java
@@ -21,7 +21,7 @@ import java.util.List;
import opennlp.tools.util.StringUtil;
-public class WordClusterFeatureGenerator extends FeatureGeneratorAdapter {
+public class WordClusterFeatureGenerator implements AdaptiveFeatureGenerator {
private WordClusterDictionary tokenDictionary;
private String resourceName;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/99323ad6/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/IdentityFeatureGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/IdentityFeatureGenerator.java b/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/IdentityFeatureGenerator.java
index 70cbd15..970fe63 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/IdentityFeatureGenerator.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/IdentityFeatureGenerator.java
@@ -19,7 +19,7 @@ package opennlp.tools.util.featuregen;
import java.util.List;
-class IdentityFeatureGenerator extends FeatureGeneratorAdapter {
+class IdentityFeatureGenerator implements AdaptiveFeatureGenerator {
public void createFeatures(List<String> features, String[] tokens, int index,
String[] previousOutcomes) {
[28/50] [abbrv] opennlp git commit: OPENNLP-871: Cleanup code base
for release
Posted by jo...@apache.org.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
index e5a9f43..22c8550 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleSequenceStream.java
@@ -70,9 +70,8 @@ public class NameSampleSequenceStream implements SequenceStream {
@SuppressWarnings("unchecked")
public Event[] updateContext(Sequence sequence, AbstractModel model) {
- Sequence<NameSample> pss = sequence;
TokenNameFinder tagger = new NameFinderME(new TokenNameFinderModel("x-unspecified", model, Collections.<String, Object>emptyMap(), null));
- String[] sentence = pss.getSource().getSentence();
+ String[] sentence = ((Sequence<NameSample>) sequence).getSource().getSentence();
String[] tags = seqCodec.encode(tagger.find(sentence), sentence.length);
Event[] events = new Event[sentence.length];
@@ -103,8 +102,7 @@ public class NameSampleSequenceStream implements SequenceStream {
events[i] = new Event(tags[i], context);
}
- Sequence<NameSample> sequence = new Sequence<NameSample>(events,sample);
- return sequence;
+ return new Sequence<>(events,sample);
}
else {
return null;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
index 7dabced..9ae1dc9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
@@ -38,12 +38,12 @@ public class NameSampleTypeFilter extends FilterObjectStream<NameSample, NameSam
public NameSampleTypeFilter(String[] types, ObjectStream<NameSample> samples) {
super(samples);
- this.types = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(types)));
+ this.types = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(types)));
}
public NameSampleTypeFilter(Set<String> types, ObjectStream<NameSample> samples) {
super(samples);
- this.types = Collections.unmodifiableSet(new HashSet<String>(types));
+ this.types = Collections.unmodifiableSet(new HashSet<>(types));
}
public NameSample read() throws IOException {
@@ -52,7 +52,7 @@ public class NameSampleTypeFilter extends FilterObjectStream<NameSample, NameSam
if (sample != null) {
- List<Span> filteredNames = new ArrayList<Span>();
+ List<Span> filteredNames = new ArrayList<>();
for (Span name : sample.getNames()) {
if (types.contains(name.getType())) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
index b72d3a9..261321b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/RegexNameFinderFactory.java
@@ -100,11 +100,11 @@ public class RegexNameFinderFactory {
}
}
- public static interface RegexAble {
+ public interface RegexAble {
- public Map<String, Pattern[]> getRegexMap();
+ Map<String, Pattern[]> getRegexMap();
- public String getType();
+ String getType();
}
public enum DEFAULT_REGEX_NAME_FINDER implements RegexAble {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java
index 48451e2..6da6f4e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinder.java
@@ -28,7 +28,7 @@ public interface TokenNameFinder {
* @param tokens an array of the tokens or words of the sequence, typically a sentence.
* @return an array of spans for each of the names identified.
*/
- public Span[] find(String tokens[]);
+ Span[] find(String tokens[]);
/**
* Forgets all adaptive data which was collected during previous
@@ -36,6 +36,6 @@ public interface TokenNameFinder {
*
* This method is typical called at the end of a document.
*/
- public void clearAdaptiveData();
+ void clearAdaptiveData();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
index fa93bce..3d2547b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
@@ -60,7 +60,7 @@ public class TokenNameFinderCrossValidator {
public DocumentSample read() throws IOException {
- List<NameSample> document = new ArrayList<NameSample>();
+ List<NameSample> document = new ArrayList<>();
if (beginSample == null) {
// Assume that the clear flag is set
@@ -143,7 +143,6 @@ public class TokenNameFinderCrossValidator {
private TokenNameFinderEvaluationMonitor[] listeners;
private FMeasure fmeasure = new FMeasure();
- private SequenceCodec<String> codec;
private TokenNameFinderFactory factory;
/**
@@ -175,7 +174,6 @@ public class TokenNameFinderCrossValidator {
this.params = trainParams;
this.listeners = listeners;
- this.codec = codec;
}
public TokenNameFinderCrossValidator(String languageCode, String type,
@@ -209,7 +207,7 @@ public class TokenNameFinderCrossValidator {
// Note: The name samples need to be grouped on a document basis.
- CrossValidationPartitioner<DocumentSample> partitioner = new CrossValidationPartitioner<DocumentSample>(
+ CrossValidationPartitioner<DocumentSample> partitioner = new CrossValidationPartitioner<>(
new NameToDocumentSampleStream(samples), nFolds);
while (partitioner.hasNext()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
index c2146f0..fb689ab 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
@@ -22,9 +22,7 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-
import opennlp.tools.cmdline.PerformanceMonitor;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.Span;
@@ -103,8 +101,7 @@ public class TokenNameFinderEvaluator extends Evaluator<NameSample> {
}
@Deprecated
- public static void main(String[] args) throws IOException,
- InvalidFormatException {
+ public static void main(String[] args) throws IOException {
if (args.length == 4) {
@@ -150,8 +147,5 @@ public class TokenNameFinderEvaluator extends Evaluator<NameSample> {
System.out.println("Recall: " + evaluator.getFMeasure().getRecallScore());
System.out.println("Precision: " + evaluator.getFMeasure().getPrecisionScore());
}
- else {
- // usage: -encoding code test.file model.file
- }
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
index 6bae7bc..a4780f5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderModel.java
@@ -116,15 +116,15 @@ public class TokenNameFinderModel extends BaseModel {
this(languageCode, nameFinderModel, null, resources, manifestInfoEntries);
}
- public TokenNameFinderModel(InputStream in) throws IOException, InvalidFormatException {
+ public TokenNameFinderModel(InputStream in) throws IOException {
super(COMPONENT_NAME, in);
}
- public TokenNameFinderModel(File modelFile) throws IOException, InvalidFormatException {
+ public TokenNameFinderModel(File modelFile) throws IOException {
super(COMPONENT_NAME, modelFile);
}
- public TokenNameFinderModel(URL modelURL) throws IOException, InvalidFormatException {
+ public TokenNameFinderModel(URL modelURL) throws IOException {
super(COMPONENT_NAME, modelURL);
}
@@ -291,14 +291,9 @@ public class TokenNameFinderModel extends BaseModel {
protected void validateArtifactMap() throws InvalidFormatException {
super.validateArtifactMap();
- if (artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof MaxentModel ||
- artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel) {
- // TODO: Check should be performed on the possible outcomes!
-// MaxentModel model = (MaxentModel) artifactMap.get(MAXENT_MODEL_ENTRY_NAME);
-// isModelValid(model);
- }
- else {
- throw new InvalidFormatException("Token Name Finder model is incomplete!");
- }
+ if (!(artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof MaxentModel) &&
+ !(artifactMap.get(MAXENT_MODEL_ENTRY_NAME) instanceof SequenceClassificationModel)) {
+ throw new InvalidFormatException("Token Name Finder model is incomplete!");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
index e4b0cbc..0e597e0 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/ngram/NGramModel.java
@@ -44,7 +44,7 @@ public class NGramModel implements Iterable<StringList>{
protected static final String COUNT = "count";
- private Map<StringList, Integer> mNGrams = new HashMap<StringList, Integer>();
+ private Map<StringList, Integer> mNGrams = new HashMap<>();
/**
* Initializes an empty instance.
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
index 3fabfd2..10c3f0e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractBottomUpParser.java
@@ -181,9 +181,9 @@ public abstract class AbstractBottomUpParser implements Parser {
reportFailedParse = true;
this.headRules = headRules;
this.punctSet = headRules.getPunctuationTags();
- odh = new ListHeap<Parse>(K);
- ndh = new ListHeap<Parse>(K);
- completeParses = new ListHeap<Parse>(K);
+ odh = new ListHeap<>(K);
+ ndh = new ListHeap<>(K);
+ completeParses = new ListHeap<>(K);
}
/**
@@ -217,7 +217,7 @@ public abstract class AbstractBottomUpParser implements Parser {
* @return An array of parses which is a subset of chunks with punctuation removed.
*/
public static Parse[] collapsePunctuation(Parse[] chunks, Set<String> punctSet) {
- List<Parse> collapsedParses = new ArrayList<Parse>(chunks.length);
+ List<Parse> collapsedParses = new ArrayList<>(chunks.length);
int lastNonPunct = -1;
int nextNonPunct;
for (int ci=0,cn=chunks.length;ci<cn;ci++) {
@@ -274,7 +274,7 @@ public abstract class AbstractBottomUpParser implements Parser {
double minComplete = 2;
double bestComplete = -100000; //approximating -infinity/0 in ln domain
while (odh.size() > 0 && (completeParses.size() < M || (odh.first()).getProb() < minComplete) && derivationStage < maxDerivationLength) {
- ndh = new ListHeap<Parse>(K);
+ ndh = new ListHeap<>(K);
int derivationRank = 0;
for (Iterator<Parse> pi = odh.iterator(); pi.hasNext() && derivationRank < K; derivationRank++) { // forearch derivation
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
index 9d72bf8..079da7e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractParserEventStream.java
@@ -66,7 +66,7 @@ public abstract class AbstractParserEventStream extends opennlp.tools.util.Abstr
@Override
protected Iterator<Event> createEvents(Parse sample) {
- List<Event> newEvents = new ArrayList<Event>();
+ List<Event> newEvents = new ArrayList<>();
Parse.pruneParse(sample);
if (fixPossesives) {
@@ -96,7 +96,7 @@ public abstract class AbstractParserEventStream extends opennlp.tools.util.Abstr
}
public static Parse[] getInitialChunks(Parse p) {
- List<Parse> chunks = new ArrayList<Parse>();
+ List<Parse> chunks = new ArrayList<>();
getInitialChunks(p, chunks);
return chunks.toArray(new Parse[chunks.size()]);
}
@@ -134,9 +134,9 @@ public abstract class AbstractParserEventStream extends opennlp.tools.util.Abstr
protected abstract void addParseEvents(List<Event> newEvents, Parse[] chunks);
private void addChunkEvents(List<Event> chunkEvents, Parse[] chunks) {
- List<String> toks = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
- List<String> preds = new ArrayList<String>();
+ List<String> toks = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
+ List<String> preds = new ArrayList<>();
for (int ci = 0, cl = chunks.length; ci < cl; ci++) {
Parse c = chunks[ci];
if (c.isPosTag()) {
@@ -168,8 +168,8 @@ public abstract class AbstractParserEventStream extends opennlp.tools.util.Abstr
}
private void addTagEvents(List<Event> tagEvents, Parse[] chunks) {
- List<String> toks = new ArrayList<String>();
- List<String> preds = new ArrayList<String>();
+ List<String> toks = new ArrayList<>();
+ List<String> preds = new ArrayList<>();
for (int ci = 0, cl = chunks.length; ci < cl; ci++) {
Parse c = chunks[ci];
if (c.isPosTag()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java
index a8b985c..2d8dae2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ChunkSampleStream.java
@@ -57,7 +57,7 @@ public class ChunkSampleStream extends FilterObjectStream<Parse, ChunkSample> {
}
public static Parse[] getInitialChunks(Parse p) {
- List<Parse> chunks = new ArrayList<Parse>();
+ List<Parse> chunks = new ArrayList<>();
getInitialChunks(p, chunks);
return chunks.toArray(new Parse[chunks.size()]);
}
@@ -68,9 +68,9 @@ public class ChunkSampleStream extends FilterObjectStream<Parse, ChunkSample> {
if (parse != null) {
Parse[] chunks = getInitialChunks(parse);
- List<String> toks = new ArrayList<String>();
- List<String> tags = new ArrayList<String>();
- List<String> preds = new ArrayList<String>();
+ List<String> toks = new ArrayList<>();
+ List<String> tags = new ArrayList<>();
+ List<String> preds = new ArrayList<>();
for (int ci = 0, cl = chunks.length; ci < cl; ci++) {
Parse c = chunks[ci];
if (c.isPosTag()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/GapLabeler.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/GapLabeler.java b/opennlp-tools/src/main/java/opennlp/tools/parser/GapLabeler.java
index 9597d52..dbc8c45 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/GapLabeler.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/GapLabeler.java
@@ -29,5 +29,5 @@ public interface GapLabeler {
* Labels the constituents found in the stack with gap labels if appropriate.
* @param stack The stack of un-completed constituents.
*/
- public void labelGaps(Stack<Constituent> stack);
+ void labelGaps(Stack<Constituent> stack);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/HeadRules.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/HeadRules.java b/opennlp-tools/src/main/java/opennlp/tools/parser/HeadRules.java
index 6993a20..204bd9a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/HeadRules.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/HeadRules.java
@@ -32,12 +32,12 @@ public interface HeadRules {
* @param type The type of a constituent which is made up of the specified constituents.
* @return The constituent which is the head.
*/
- public Parse getHead(Parse[] constituents, String type);
+ Parse getHead(Parse[] constituents, String type);
/**
* Returns the set of punctuation tags. Attachment decisions for these tags will not be modeled.
*
* @return the set of punctuation tags.
*/
- public Set<String> getPunctuationTags();
+ Set<String> getPunctuationTags();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java b/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
index 9bf12f3..ce3610b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/Parse.java
@@ -25,6 +25,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Objects;
import java.util.Set;
import java.util.Stack;
import java.util.TreeSet;
@@ -153,7 +154,7 @@ public class Parse implements Cloneable, Comparable<Parse> {
this.prob = p;
this.head = this;
this.headIndex = index;
- this.parts = new LinkedList<Parse>();
+ this.parts = new LinkedList<>();
this.label = null;
this.parent = null;
}
@@ -179,7 +180,7 @@ public class Parse implements Cloneable, Comparable<Parse> {
@Override
public Object clone() {
Parse p = new Parse(this.text, this.span, this.type, this.prob, this.head);
- p.parts = new LinkedList<Parse>();
+ p.parts = new LinkedList<>();
p.parts.addAll(this.parts);
if (derivation != null) {
@@ -843,8 +844,8 @@ public class Parse implements Cloneable, Comparable<Parse> {
public static Parse parseParse(String parse, GapLabeler gl) {
StringBuilder text = new StringBuilder();
int offset = 0;
- Stack<Constituent> stack = new Stack<Constituent>();
- List<Constituent> cons = new LinkedList<Constituent>();
+ Stack<Constituent> stack = new Stack<>();
+ List<Constituent> cons = new LinkedList<>();
for (int ci = 0, cl = parse.length(); ci < cl; ci++) {
char c = parse.charAt(ci);
if (c == '(') {
@@ -856,7 +857,7 @@ public class Parse implements Cloneable, Comparable<Parse> {
String token = getToken(rest);
stack.push(new Constituent(type, new Span(offset,offset)));
if (token != null) {
- if (type.equals("-NONE-") && gl != null) {
+ if (Objects.equals(type, "-NONE-") && gl != null) {
//System.err.println("stack.size="+stack.size());
gl.labelGaps(stack);
}
@@ -950,8 +951,8 @@ public class Parse implements Cloneable, Comparable<Parse> {
* @return the parse nodes which are children of this node and which are pos tags.
*/
public Parse[] getTagNodes() {
- List<Parse> tags = new LinkedList<Parse>();
- List<Parse> nodes = new LinkedList<Parse>();
+ List<Parse> tags = new LinkedList<>();
+ List<Parse> nodes = new LinkedList<>();
nodes.addAll(this.parts);
while(nodes.size() != 0) {
Parse p = nodes.remove(0);
@@ -978,7 +979,7 @@ public class Parse implements Cloneable, Comparable<Parse> {
if (this == node) {
return parent;
}
- Set<Parse> parents = new HashSet<Parse>();
+ Set<Parse> parents = new HashSet<>();
Parse cparent = this;
while(cparent != null) {
parents.add(cparent);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/Parser.java
index 0a7152b..64964f2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/Parser.java
@@ -34,13 +34,13 @@ public interface Parser {
* @param numParses The number of parses desired.
* @return the specified number of parses for the specified tokens.
*/
- public abstract Parse[] parse(Parse tokens, int numParses);
+ Parse[] parse(Parse tokens, int numParses);
/**
* Returns a parse for the specified parse of tokens.
* @param tokens The root node of a flat parse containing only tokens.
* @return A full parse of the specified tokens or the flat chunks of the tokens if a fullparse could not be found.
*/
- public abstract Parse parse(Parse tokens);
+ Parse parse(Parse tokens);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerFactory.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerFactory.java
index 87e7af8..b19d480 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerFactory.java
@@ -33,7 +33,7 @@ public class ParserChunkerFactory extends ChunkerFactory {
@Override
public SequenceValidator<String> getSequenceValidator() {
- MaxentModel model = (MaxentModel) artifactProvider.getArtifact("chunker.model");
+ MaxentModel model = artifactProvider.getArtifact("chunker.model");
String outcomes[] = new String[model.getNumOutcomes()];
for (int i = 0; i < outcomes.length; i++) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerSequenceValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerSequenceValidator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerSequenceValidator.java
index b507a4e..9cba697 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerSequenceValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserChunkerSequenceValidator.java
@@ -29,8 +29,7 @@ public class ParserChunkerSequenceValidator implements SequenceValidator<String>
public ParserChunkerSequenceValidator(String outcomes[]) {
- continueStartMap =
- new HashMap<String, String>(outcomes.length);
+ continueStartMap = new HashMap<>(outcomes.length);
for (int oi=0, on = outcomes.length; oi<on; oi++) {
String outcome = outcomes[oi];
if (outcome.startsWith(Parser.CONT)){
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserCrossValidator.java
index 938004c..f50f277 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserCrossValidator.java
@@ -48,12 +48,10 @@ public class ParserCrossValidator {
public void evaluate(ObjectStream<Parse> samples, int nFolds) throws IOException {
- CrossValidationPartitioner<Parse> partitioner = new CrossValidationPartitioner<Parse>(
- samples, nFolds);
+ CrossValidationPartitioner<Parse> partitioner = new CrossValidationPartitioner<>(samples, nFolds);
while (partitioner.hasNext()) {
- CrossValidationPartitioner.TrainingSampleStream<Parse> trainingSampleStream = partitioner
- .next();
+ CrossValidationPartitioner.TrainingSampleStream<Parse> trainingSampleStream = partitioner.next();
ParserModel model;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java
index 4bddbe5..97dff6c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java
@@ -63,14 +63,14 @@ public class ParserEvaluator extends Evaluator<Parse> {
*/
private static Span[] getConstituencySpans(final Parse parse) {
- Stack<Parse> stack = new Stack<Parse>();
+ Stack<Parse> stack = new Stack<>();
if (parse.getChildCount() > 0) {
for (Parse child : parse.getChildren()) {
stack.push(child);
}
}
- List<Span> consts = new ArrayList<Span>();
+ List<Span> consts = new ArrayList<>();
while (!stack.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java
index 84d34da..d3802e2 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/ParserModel.java
@@ -47,8 +47,7 @@ public class ParserModel extends BaseModel {
private static class POSModelSerializer implements ArtifactSerializer<POSModel> {
- public POSModel create(InputStream in) throws IOException,
- InvalidFormatException {
+ public POSModel create(InputStream in) throws IOException {
POSModel posModel = new POSModel(new UncloseableInputStream(in));
// The 1.6.x models write the non-default beam size into the model itself.
@@ -73,8 +72,7 @@ public class ParserModel extends BaseModel {
private static class ChunkerModelSerializer implements ArtifactSerializer<ChunkerModel> {
- public ChunkerModel create(InputStream in) throws IOException,
- InvalidFormatException {
+ public ChunkerModel create(InputStream in) throws IOException {
ChunkerModel model = new ChunkerModel(new UncloseableInputStream(in));
@@ -176,15 +174,15 @@ public class ParserModel extends BaseModel {
chunkerTagger, headRules, type, manifestInfoEntries);
}
- public ParserModel(InputStream in) throws IOException, InvalidFormatException {
+ public ParserModel(InputStream in) throws IOException {
super(COMPONENT_NAME, in);
}
- public ParserModel(File modelFile) throws IOException, InvalidFormatException {
+ public ParserModel(File modelFile) throws IOException {
super(COMPONENT_NAME, modelFile);
}
- public ParserModel(URL modelURL) throws IOException, InvalidFormatException {
+ public ParserModel(URL modelURL) throws IOException {
super(COMPONENT_NAME, modelURL);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
index 42d7c07..bff687d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/BuildContextGenerator.java
@@ -68,7 +68,7 @@ public class BuildContextGenerator extends AbstractContextGenerator {
* @return the context for building constituents at the specified index.
*/
public String[] getContext(Parse[] constituents, int index) {
- List<String> features = new ArrayList<String>(100);
+ List<String> features = new ArrayList<>(100);
int ps = constituents.length;
// cons(-2), cons(-1), cons(0), cons(1), cons(2)
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/CheckContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/CheckContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/CheckContextGenerator.java
index 6dcaf08..b8591c8 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/CheckContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/CheckContextGenerator.java
@@ -53,7 +53,7 @@ public class CheckContextGenerator extends AbstractContextGenerator {
*/
public String[] getContext(Parse[] constituents, String type, int start, int end) {
int ps = constituents.length;
- List<String> features = new ArrayList<String>(100);
+ List<String> features = new ArrayList<>(100);
//default
features.add("default");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
index 062e7f2..ae20e8a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/Parser.java
@@ -104,8 +104,8 @@ public class Parser extends AbstractBottomUpParser {
cprobs = new double[checkModel.getNumOutcomes()];
this.buildContextGenerator = new BuildContextGenerator();
this.checkContextGenerator = new CheckContextGenerator();
- startTypeMap = new HashMap<String, String>();
- contTypeMap = new HashMap<String, String>();
+ startTypeMap = new HashMap<>();
+ contTypeMap = new HashMap<>();
for (int boi = 0, bon = buildModel.getNumOutcomes(); boi < bon; boi++) {
String outcome = buildModel.getOutcome(boi);
if (outcome.startsWith(START)) {
@@ -134,15 +134,15 @@ public class Parser extends AbstractBottomUpParser {
@Override
protected Parse[] advanceParses(final Parse p, double probMass) {
double q = 1 - probMass;
- /** The closest previous node which has been labeled as a start node. */
+ /* The closest previous node which has been labeled as a start node. */
Parse lastStartNode = null;
- /** The index of the closest previous node which has been labeled as a start node. */
+ /* The index of the closest previous node which has been labeled as a start node. */
int lastStartIndex = -1;
- /** The type of the closest previous node which has been labeled as a start node. */
+ /* The type of the closest previous node which has been labeled as a start node. */
String lastStartType = null;
- /** The index of the node which will be labeled in this iteration of advancing the parse. */
+ /* The index of the node which will be labeled in this iteration of advancing the parse. */
int advanceNodeIndex;
- /** The node which will be labeled in this iteration of advancing the parse. */
+ /* The node which will be labeled in this iteration of advancing the parse. */
Parse advanceNode=null;
Parse[] originalChildren = p.getChildren();
Parse[] children = collapsePunctuation(originalChildren,punctSet);
@@ -164,7 +164,7 @@ public class Parser extends AbstractBottomUpParser {
}
}
int originalAdvanceIndex = mapParseIndex(advanceNodeIndex,children,originalChildren);
- List<Parse> newParsesList = new ArrayList<Parse>(buildModel.getNumOutcomes());
+ List<Parse> newParsesList = new ArrayList<>(buildModel.getNumOutcomes());
//call build
buildModel.eval(buildContextGenerator.getContext(children, advanceNodeIndex), bprobs);
double bprobSum = 0;
@@ -274,12 +274,12 @@ public class Parser extends AbstractBottomUpParser {
parseSamples.reset();
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
+ Map<String, String> manifestInfoEntries = new HashMap<>();
// build
System.err.println("Training builder");
ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict);
- Map<String, String> buildReportMap = new HashMap<String, String>();
+ Map<String, String> buildReportMap = new HashMap<>();
EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getSettings("build"), buildReportMap);
MaxentModel buildModel = buildTrainer.train(bes);
mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build");
@@ -308,7 +308,7 @@ public class Parser extends AbstractBottomUpParser {
// check
System.err.println("Training checker");
ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK);
- Map<String, String> checkReportMap = new HashMap<String, String>();
+ Map<String, String> checkReportMap = new HashMap<>();
EventTrainer checkTrainer = TrainerFactory.getEventTrainer( mlParams.getSettings("check"), checkReportMap);
MaxentModel checkModel = checkTrainer.train(kes);
mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
index 88c4e24..afaed99 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/chunking/ParserEventStream.java
@@ -93,7 +93,7 @@ public class ParserEventStream extends AbstractParserEventStream {
if (!type.equals(AbstractBottomUpParser.TOP_NODE)) {
reducedChunks = new Parse[chunks.length-(reduceEnd-reduceStart+1)+1]; //total - num_removed + 1 (for new node)
//insert nodes before reduction
- for (int ri=0,rn=reduceStart;ri<rn;ri++) {
+ for (int ri = 0; ri< reduceStart; ri++) {
reducedChunks[ri]=chunks[ri];
}
//insert reduced node
@@ -177,24 +177,25 @@ public class ParserEventStream extends AbstractParserEventStream {
boolean fun = false;
int ai = 0;
while (ai < args.length && args[ai].startsWith("-")) {
- if (args[ai].equals("-build")) {
- etype = ParserEventTypeEnum.BUILD;
- }
- else if (args[ai].equals("-check")) {
- etype = ParserEventTypeEnum.CHECK;
- }
- else if (args[ai].equals("-chunk")) {
- etype = ParserEventTypeEnum.CHUNK;
- }
- else if (args[ai].equals("-tag")) {
- etype = ParserEventTypeEnum.TAG;
- }
- else if (args[ai].equals("-fun")) {
- fun = true;
- }
- else {
- System.err.println("Invalid option " + args[ai]);
- System.exit(1);
+ switch (args[ai]) {
+ case "-build":
+ etype = ParserEventTypeEnum.BUILD;
+ break;
+ case "-check":
+ etype = ParserEventTypeEnum.CHECK;
+ break;
+ case "-chunk":
+ etype = ParserEventTypeEnum.CHUNK;
+ break;
+ case "-tag":
+ etype = ParserEventTypeEnum.TAG;
+ break;
+ case "-fun":
+ fun = true;
+ break;
+ default:
+ System.err.println("Invalid option " + args[ai]);
+ System.exit(1);
}
ai++;
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
index 8a5a947..86b74cc 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/en/HeadRules.java
@@ -34,12 +34,10 @@ import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.StringTokenizer;
-
import opennlp.tools.parser.Constituent;
import opennlp.tools.parser.GapLabeler;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.chunking.Parser;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.model.ArtifactSerializer;
import opennlp.tools.util.model.SerializableArtifact;
@@ -50,8 +48,7 @@ public class HeadRules implements opennlp.tools.parser.HeadRules, GapLabeler, Se
public static class HeadRulesSerializer implements ArtifactSerializer<opennlp.tools.parser.lang.en.HeadRules> {
- public opennlp.tools.parser.lang.en.HeadRules create(InputStream in) throws IOException,
- InvalidFormatException {
+ public opennlp.tools.parser.lang.en.HeadRules create(InputStream in) throws IOException {
return new opennlp.tools.parser.lang.en.HeadRules(new BufferedReader(new InputStreamReader(in, "UTF-8")));
}
@@ -118,7 +115,7 @@ public class HeadRules implements opennlp.tools.parser.HeadRules, GapLabeler, Se
BufferedReader in = new BufferedReader(rulesReader);
readHeadRules(in);
- punctSet = new HashSet<String>();
+ punctSet = new HashSet<>();
punctSet.add(".");
punctSet.add(",");
punctSet.add("``");
@@ -197,7 +194,7 @@ public class HeadRules implements opennlp.tools.parser.HeadRules, GapLabeler, Se
private void readHeadRules(BufferedReader str) throws IOException {
String line;
- headRules = new HashMap<String, HeadRule>(30);
+ headRules = new HashMap<>(30);
while ((line = str.readLine()) != null) {
StringTokenizer st = new StringTokenizer(line);
String num = st.nextToken();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
index 946fa5c..6f5e32a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/lang/es/AncoraSpanishHeadRules.java
@@ -33,12 +33,10 @@ import java.util.Map;
import java.util.Set;
import java.util.Stack;
import java.util.StringTokenizer;
-
import opennlp.tools.parser.Constituent;
import opennlp.tools.parser.GapLabeler;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.chunking.Parser;
-import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.model.ArtifactSerializer;
import opennlp.tools.util.model.SerializableArtifact;
@@ -62,8 +60,7 @@ public class AncoraSpanishHeadRules implements opennlp.tools.parser.HeadRules, G
public static class HeadRulesSerializer implements ArtifactSerializer<opennlp.tools.parser.lang.es.AncoraSpanishHeadRules> {
- public opennlp.tools.parser.lang.es.AncoraSpanishHeadRules create(InputStream in) throws IOException,
- InvalidFormatException {
+ public opennlp.tools.parser.lang.es.AncoraSpanishHeadRules create(InputStream in) throws IOException {
return new opennlp.tools.parser.lang.es.AncoraSpanishHeadRules(new BufferedReader(new InputStreamReader(in, "UTF-8")));
}
@@ -119,7 +116,7 @@ public class AncoraSpanishHeadRules implements opennlp.tools.parser.HeadRules, G
BufferedReader in = new BufferedReader(rulesReader);
readHeadRules(in);
- punctSet = new HashSet<String>();
+ punctSet = new HashSet<>();
punctSet.add(".");
punctSet.add(",");
punctSet.add("``");
@@ -199,7 +196,7 @@ public class AncoraSpanishHeadRules implements opennlp.tools.parser.HeadRules, G
private void readHeadRules(BufferedReader str) throws IOException {
String line;
- headRules = new HashMap<String, HeadRule>(60);
+ headRules = new HashMap<>(60);
while ((line = str.readLine()) != null) {
StringTokenizer st = new StringTokenizer(line);
String num = st.nextToken();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
index 0d82e0b..3fdeb44 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/DefaultPOSSequenceValidator.java
@@ -35,11 +35,7 @@ public class DefaultPOSSequenceValidator implements SequenceValidator<String> {
return true;
} else {
String[] tags = tagDictionary.getTags(inputSequence[i]);
- if (tags == null) {
- return true;
- } else {
- return Arrays.asList(tags).contains(outcome);
- }
+ return tags == null || Arrays.asList(tags).contains(outcome);
}
}
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java b/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java
index 1e52444..52df16d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/MutableTagDictionary.java
@@ -36,7 +36,7 @@ public interface MutableTagDictionary extends TagDictionary {
* @return the previous tags associated with the word, or null if there was no
* mapping for word.
*/
- public String[] put(String word, String... tags);
+ String[] put(String word, String... tags);
/**
* Whether if the dictionary is case sensitive or not
@@ -45,6 +45,6 @@ public interface MutableTagDictionary extends TagDictionary {
*/
// TODO: move to TagDictionary, can't do it now because of backward
// compatibility.
- public boolean isCaseSensitive();
+ boolean isCaseSensitive();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/postag/POSContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSContextGenerator.java
index 6f3e831..29c90d5 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSContextGenerator.java
@@ -25,5 +25,5 @@ import opennlp.tools.util.BeamSearchContextGenerator;
* The interface for a context generator for the POS Tagger.
*/
public interface POSContextGenerator extends BeamSearchContextGenerator<String> {
- public String[] getContext(int pos, String[] tokens, String[] prevTags, Object[] ac);
+ String[] getContext(int pos, String[] tokens, String[] prevTags, Object[] ac);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
index f081916..bc334ea 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagger.java
@@ -17,8 +17,6 @@
package opennlp.tools.postag;
-import java.util.List;
-
import opennlp.tools.util.Sequence;
/**
@@ -31,11 +29,11 @@ public interface POSTagger {
* @param sentence The sentece of tokens to be tagged.
* @return an array of pos tags for each token provided in sentence.
*/
- public String[] tag(String[] sentence);
+ String[] tag(String[] sentence);
- public String[] tag(String[] sentence, Object[] additionaContext);
+ String[] tag(String[] sentence, Object[] additionaContext);
- public Sequence[] topKSequences(String[] sentence);
+ Sequence[] topKSequences(String[] sentence);
- public Sequence[] topKSequences(String[] sentence, Object[] additionaContext);
+ Sequence[] topKSequences(String[] sentence, Object[] additionaContext);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
index 27854dc..b4e5d12 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
@@ -90,7 +90,7 @@ public class POSTaggerCrossValidator {
*/
public void evaluate(ObjectStream<POSSample> samples, int nFolds) throws IOException {
- CrossValidationPartitioner<POSSample> partitioner = new CrossValidationPartitioner<POSSample>(
+ CrossValidationPartitioner<POSSample> partitioner = new CrossValidationPartitioner<>(
samples, nFolds);
while (partitioner.hasNext()) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/postag/TagDictionary.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/TagDictionary.java b/opennlp-tools/src/main/java/opennlp/tools/postag/TagDictionary.java
index cdd3db9..1e31d8a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/TagDictionary.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/TagDictionary.java
@@ -31,5 +31,5 @@ public interface TagDictionary {
* @return A list of valid tags for the specified word or null if no information
* is available for that word.
*/
- public String[] getTags(String word);
+ String[] getTags(String word);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/EndOfSentenceScanner.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/EndOfSentenceScanner.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/EndOfSentenceScanner.java
index 968689f..3e3ab42 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/EndOfSentenceScanner.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/EndOfSentenceScanner.java
@@ -36,7 +36,7 @@ public interface EndOfSentenceScanner {
* Returns an array of character which can indicate the end of a sentence.
* @return an array of character which can indicate the end of a sentence.
*/
- public char[] getEndOfSentenceCharacters();
+ char[] getEndOfSentenceCharacters();
/**
* The receiver scans the specified string for sentence ending characters and
@@ -45,7 +45,7 @@ public interface EndOfSentenceScanner {
* @param s a <code>String</code> value
* @return a <code>List</code> of Integer objects.
*/
- public List<Integer> getPositions(String s);
+ List<Integer> getPositions(String s);
/**
* The receiver scans `buf' for sentence ending characters and
@@ -54,7 +54,7 @@ public interface EndOfSentenceScanner {
* @param buf a <code>StringBuffer</code> value
* @return a <code>List</code> of Integer objects.
*/
- public List<Integer> getPositions(StringBuffer buf);
+ List<Integer> getPositions(StringBuffer buf);
/**
* The receiver scans `cbuf' for sentence ending characters and
@@ -63,5 +63,5 @@ public interface EndOfSentenceScanner {
* @param cbuf a <code>char[]</code> value
* @return a <code>List</code> of Integer objects.
*/
- public List<Integer> getPositions(char[] cbuf);
+ List<Integer> getPositions(char[] cbuf);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDContextGenerator.java
index 16989eb..ade9d6a 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDContextGenerator.java
@@ -33,5 +33,5 @@ public interface SDContextGenerator {
* @return an array of contextual features for the potential sentence boundary at the
* specified position within the specified string buffer.
*/
- public abstract String[] getContext(CharSequence s, int position);
+ String[] getContext(CharSequence s, int position);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetector.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetector.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetector.java
index dc8649b..01e3d99 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetector.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetector.java
@@ -33,7 +33,7 @@ public interface SentenceDetector {
* @return The String[] with the individual sentences as the array
* elements.
*/
- public String[] sentDetect(String s);
+ String[] sentDetect(String s);
/**
* Sentence detect a string.
@@ -43,5 +43,5 @@ public interface SentenceDetector {
* @return The Span[] with the spans (offsets into s) for each
* detected sentence as the individuals array elements.
*/
- public Span[] sentPosDetect(String s);
+ Span[] sentPosDetect(String s);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/stemmer/PorterStemmer.java b/opennlp-tools/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
index 28688df..e125df7 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/stemmer/PorterStemmer.java
@@ -79,7 +79,8 @@ public class PorterStemmer implements Stemmer {
if (b.length == i) {
char[] new_b = new char[i+INC];
- for (int c = 0; c < i; c++) new_b[c] = b[c]; {
+ System.arraycopy(b, 0, new_b, 0, i);
+ {
b = new_b;
}
}
@@ -108,12 +109,12 @@ public class PorterStemmer implements Stemmer {
/* cons(i) is true <=> b[i] is a consonant. */
- private final boolean cons(int i) {
+ private boolean cons(int i) {
switch (b[i]) {
case 'a': case 'e': case 'i': case 'o': case 'u':
return false;
case 'y':
- return (i==k0) ? true : !cons(i-1);
+ return (i == k0) || !cons(i - 1);
default:
return true;
}
@@ -130,7 +131,7 @@ public class PorterStemmer implements Stemmer {
....
*/
- private final int m() {
+ private int m() {
int n = 0;
int i = k0;
while(true) {
@@ -164,7 +165,7 @@ public class PorterStemmer implements Stemmer {
/* vowelinstem() is true <=> k0,...j contains a vowel */
- private final boolean vowelinstem() {
+ private boolean vowelinstem() {
int i;
for (i = k0; i <= j; i++)
if (! cons(i))
@@ -174,12 +175,8 @@ public class PorterStemmer implements Stemmer {
/* doublec(j) is true <=> j,(j-1) contain a double consonant. */
- private final boolean doublec(int j) {
- if (j < k0+1)
- return false;
- if (b[j] != b[j-1])
- return false;
- return cons(j);
+ private boolean doublec(int j) {
+ return j >= k0 + 1 && b[j] == b[j - 1] && cons(j);
}
/* cvc(i) is true <=> i-2,i-1,i has the form consonant - vowel - consonant
@@ -191,7 +188,7 @@ public class PorterStemmer implements Stemmer {
*/
- private final boolean cvc(int i) {
+ private boolean cvc(int i) {
if (i < k0+2 || !cons(i) || cons(i-1) || !cons(i-2))
return false;
else {
@@ -201,7 +198,7 @@ public class PorterStemmer implements Stemmer {
return true;
}
- private final boolean ends(String s) {
+ private boolean ends(String s) {
int l = s.length();
int o = k-l+1;
if (o < k0)
@@ -251,7 +248,7 @@ public class PorterStemmer implements Stemmer {
*/
- private final void step1() {
+ private void step1() {
if (b[k] == 's') {
if (ends("sses")) k -= 2;
else if (ends("ies")) setto("i");
@@ -278,7 +275,7 @@ public class PorterStemmer implements Stemmer {
/* step2() turns terminal y to i when there is another vowel in the stem. */
- private final void step2() {
+ private void step2() {
if (ends("y") && vowelinstem()) {
b[k] = 'i';
dirty = true;
@@ -289,7 +286,7 @@ public class PorterStemmer implements Stemmer {
-ation) maps to -ize etc. note that the string before the suffix must give
m() > 0. */
- private final void step3() {
+ private void step3() {
if (k == k0) return; /* For Bug 1 */
switch (b[k-1]) {
case 'a':
@@ -333,7 +330,7 @@ public class PorterStemmer implements Stemmer {
/* step4() deals with -ic-, -full, -ness etc. similar strategy to step3. */
- private final void step4() {
+ private void step4() {
switch (b[k]) {
case 'e':
if (ends("icate")) { r("ic"); break; }
@@ -355,7 +352,7 @@ public class PorterStemmer implements Stemmer {
/* step5() takes off -ant, -ence etc., in context <c>vcvc<v>. */
- private final void step5() {
+ private void step5() {
if (k == k0) return; /* for Bug 1 */
switch (b[k-1]) {
case 'a':
@@ -410,7 +407,7 @@ public class PorterStemmer implements Stemmer {
/* step6() removes a final -e if m() > 1. */
- private final void step6() {
+ private void step6() {
j = k;
if (b[k] == 'e') {
int a = m();
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/stemmer/Stemmer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/stemmer/Stemmer.java b/opennlp-tools/src/main/java/opennlp/tools/stemmer/Stemmer.java
index eab6695..750890e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/stemmer/Stemmer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/stemmer/Stemmer.java
@@ -22,5 +22,5 @@ package opennlp.tools.stemmer;
*/
public interface Stemmer {
- public CharSequence stem(CharSequence word);
+ CharSequence stem(CharSequence word);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenContextGenerator.java
index 863a04f..b15fd91 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenContextGenerator.java
@@ -31,5 +31,5 @@ public interface TokenContextGenerator {
* @return an array of features for the specified sentence string at the
* specified index.
*/
- public abstract String[] getContext(String sentence, int index);
+ String[] getContext(String sentence, int index);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/tokenize/Tokenizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/tokenize/Tokenizer.java b/opennlp-tools/src/main/java/opennlp/tools/tokenize/Tokenizer.java
index 0ab4bc3..aae4a83 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/tokenize/Tokenizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/tokenize/Tokenizer.java
@@ -52,7 +52,7 @@ public interface Tokenizer {
* @return The String[] with the individual tokens as the array
* elements.
*/
- public String[] tokenize(String s);
+ String[] tokenize(String s);
/**
* Finds the boundaries of atomic parts in a string.
@@ -61,5 +61,5 @@ public interface Tokenizer {
* @return The Span[] with the spans (offsets into s) for each
* token as the individuals array elements.
*/
- public Span[] tokenizePos(String s);
+ Span[] tokenizePos(String s);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearchContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearchContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearchContextGenerator.java
index 5cdc6ab..4367db6 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearchContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/BeamSearchContextGenerator.java
@@ -29,5 +29,5 @@ public interface BeamSearchContextGenerator<T> {
* @param additionalContext Any addition context specific to a class implementing this interface.
* @return the context for the specified position in the specified sequence.
*/
- public String[] getContext(int index, T[] sequence, String[] priorDecisions, Object[] additionalContext);
+ String[] getContext(int index, T[] sequence, String[] priorDecisions, Object[] additionalContext);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/util/Heap.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/Heap.java b/opennlp-tools/src/main/java/opennlp/tools/util/Heap.java
index 5ca8409..a27f333 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/Heap.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/Heap.java
@@ -31,47 +31,47 @@ public interface Heap<E> {
* Removes the smallest element from the heap and returns it.
* @return The smallest element from the heap.
*/
- public E extract();
+ E extract();
/**
* Returns the smallest element of the heap.
* @return The top element of the heap.
*/
- public E first();
+ E first();
/**
* Returns the largest element of the heap.
* @return The largest element of the heap.
*/
- public E last();
+ E last();
/**
* Adds the specified object to the heap.
* @param o The object to add to the heap.
*/
- public void add(E o);
+ void add(E o);
/**
* Returns the size of the heap.
* @return The size of the heap.
*/
- public int size();
+ int size();
/**
* Returns whether the heap is empty.
* @return true if the heap is empty; false otherwise.
*/
- public boolean isEmpty();
+ boolean isEmpty();
/**
* Returns an iterator over the elements of the heap. No specific ordering of these
* elements is guaranteed.
* @return An iterator over the elements of the heap.
*/
- public Iterator<E> iterator();
+ Iterator<E> iterator();
/**
* Clears the contents of the heap.
*/
- public void clear();
+ void clear();
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/util/TrainingParameters.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/TrainingParameters.java b/opennlp-tools/src/main/java/opennlp/tools/util/TrainingParameters.java
index 6295c14..3677fc4 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/TrainingParameters.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/TrainingParameters.java
@@ -37,7 +37,7 @@ public class TrainingParameters {
public static final String CUTOFF_PARAM = "Cutoff";
public static final String THREADS_PARAM = "Threads";
- private Map<String, String> parameters = new HashMap<String, String>();
+ private Map<String, String> parameters = new HashMap<>();
public TrainingParameters() {
}
@@ -79,7 +79,7 @@ public class TrainingParameters {
*/
public Map<String, String> getSettings(String namespace) {
- Map<String, String> trainingParams = new HashMap<String, String>();
+ Map<String, String> trainingParams = new HashMap<>();
for (Map.Entry<String, String> entry : parameters.entrySet()) {
String key = entry.getKey();
@@ -146,7 +146,7 @@ public class TrainingParameters {
properties.store(out, null);
}
- public static final TrainingParameters defaultParams() {
+ public static TrainingParameters defaultParams() {
TrainingParameters mlParams = new TrainingParameters();
mlParams.put(TrainingParameters.ALGORITHM_PARAM, "MAXENT");
mlParams.put(TrainingParameters.TRAINER_TYPE_PARAM, EventTrainer.EVENT_VALUE);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/afc6b65b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
index 9997a82..6e84715 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/model/BaseModel.java
@@ -50,8 +50,6 @@ import opennlp.tools.util.ext.ExtensionLoader;
*/
public abstract class BaseModel implements ArtifactProvider, Serializable {
- private static int MODEL_BUFFER_SIZE_LIMIT = Integer.MAX_VALUE;
-
protected static final String MANIFEST_ENTRY = "manifest.properties";
protected static final String FACTORY_NAME = "factory";
@@ -173,15 +171,14 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
* @param in the input stream containing the model
*
* @throws IOException
- * @throws InvalidFormatException
*/
- protected BaseModel(String componentName, InputStream in) throws IOException, InvalidFormatException {
+ protected BaseModel(String componentName, InputStream in) throws IOException {
this(componentName, true);
loadModel(in);
}
- protected BaseModel(String componentName, File modelFile) throws IOException, InvalidFormatException {
+ protected BaseModel(String componentName, File modelFile) throws IOException {
this(componentName, true);
try (InputStream in = new BufferedInputStream(new FileInputStream(modelFile))) {
@@ -189,7 +186,7 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
}
}
- protected BaseModel(String componentName, URL modelURL) throws IOException, InvalidFormatException {
+ protected BaseModel(String componentName, URL modelURL) throws IOException {
this(componentName, true);
try (InputStream in = new BufferedInputStream(modelURL.openStream())) {
@@ -197,7 +194,7 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
}
}
- private void loadModel(InputStream in) throws IOException, InvalidFormatException {
+ private void loadModel(InputStream in) throws IOException {
if (in == null) {
throw new IllegalArgumentException("in must not be null!");
@@ -210,6 +207,7 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
}
// TODO: Discuss this solution, the buffering should
+ int MODEL_BUFFER_SIZE_LIMIT = Integer.MAX_VALUE;
in.mark(MODEL_BUFFER_SIZE_LIMIT);
final ZipInputStream zip = new ZipInputStream(in);
@@ -290,11 +288,11 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
* Finish loading the artifacts now that it knows all serializers.
*/
private void finishLoadingArtifacts(InputStream in)
- throws InvalidFormatException, IOException {
+ throws IOException {
final ZipInputStream zip = new ZipInputStream(in);
- Map<String, Object> artifactMap = new HashMap<String, Object>();
+ Map<String, Object> artifactMap = new HashMap<>();
ZipEntry entry;
while((entry = zip.getNextEntry()) != null ) {
@@ -355,7 +353,7 @@ public abstract class BaseModel implements ArtifactProvider, Serializable {
}
protected static Map<String, ArtifactSerializer> createArtifactSerializers() {
- Map<String, ArtifactSerializer> serializers = new HashMap<String, ArtifactSerializer>();
+ Map<String, ArtifactSerializer> serializers = new HashMap<>();
GenericModelSerializer.register(serializers);
PropertiesSerializer.register(serializers);
[18/50] [abbrv] opennlp git commit: Replace StringBuilder with String
Posted by jo...@apache.org.
Replace StringBuilder with String
Using a String concatenation makes the code shorter and simpler.
See issue OPENNLP-871
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c3eeb5c5
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c3eeb5c5
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c3eeb5c5
Branch: refs/heads/889
Commit: c3eeb5c530e9b0e738f84e74c774b4f63d944a7f
Parents: 7ef7235
Author: J�rn Kottmann <jo...@apache.org>
Authored: Wed Nov 2 19:57:47 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Mon Dec 19 23:37:33 2016 +0100
----------------------------------------------------------------------
.../java/opennlp/tools/cmdline/GenerateManualTool.java | 6 ++----
.../opennlp/tools/parser/AbstractContextGenerator.java | 10 ++--------
.../tools/sentdetect/DefaultSDContextGenerator.java | 8 ++++----
.../opennlp/tools/sentdetect/SentenceDetectorFactory.java | 4 +---
4 files changed, 9 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c3eeb5c5/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
index ed58dde..b42f69c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
@@ -76,10 +76,8 @@ public class GenerateManualTool {
* @return this tool usage
*/
private static String getUsage() {
- StringBuilder sb = new StringBuilder();
- sb.append("Requires one argument: \n");
- sb.append(" Path to the output XML file \n");
- return sb.toString();
+ return "Requires one argument: \n" +
+ " Path to the output XML file \n";
}
/**
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c3eeb5c5/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
index f537576..09d3b0f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/parser/AbstractContextGenerator.java
@@ -42,10 +42,7 @@ public abstract class AbstractContextGenerator {
* @return Punctuation feature for the specified parse and the specified punctuation at the specfied index.
*/
protected String punct(Parse punct, int i) {
- StringBuilder feat = new StringBuilder(5);
- feat.append(i).append("=");
- feat.append(punct.getCoveredText());
- return feat.toString();
+ return String.valueOf(i) + "=" + punct.getCoveredText();
}
/**
@@ -55,10 +52,7 @@ public abstract class AbstractContextGenerator {
* @return Punctuation feature for the specified parse and the specified punctuation at the specfied index.
*/
protected String punctbo(Parse punct, int i) {
- StringBuilder feat = new StringBuilder(5);
- feat.append(i).append("=");
- feat.append(punct.getType());
- return feat.toString();
+ return String.valueOf(i) + "=" + punct.getType();
}
protected String cons(Parse p, int i) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c3eeb5c5/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java
index 7559a14..822c43b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/DefaultSDContextGenerator.java
@@ -129,10 +129,10 @@ public class DefaultSDContextGenerator implements SDContextGenerator {
}
}
}
- prefix = new StringBuffer(sb.subSequence(prefixStart, position)).toString().trim();
+ prefix = String.valueOf(sb.subSequence(prefixStart, position)).trim();
}
int prevStart = previousSpaceIndex(sb, prefixStart);
- previous = new StringBuffer(sb.subSequence(prevStart, prefixStart)).toString().trim();
+ previous = String.valueOf(sb.subSequence(prevStart, prefixStart)).trim();
int suffixEnd = nextSpaceIndex(sb, position, lastIndex);
{
@@ -153,8 +153,8 @@ public class DefaultSDContextGenerator implements SDContextGenerator {
next = "";
}
else {
- suffix = new StringBuilder(sb.subSequence(position + 1, suffixEnd)).toString().trim();
- next = new StringBuilder(sb.subSequence(suffixEnd + 1, nextEnd)).toString().trim();
+ suffix = String.valueOf(sb.subSequence(position + 1, suffixEnd)).trim();
+ next = String.valueOf(sb.subSequence(suffixEnd + 1, nextEnd)).trim();
}
collectFeatures(prefix,suffix,previous,next, sb.charAt(position));
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c3eeb5c5/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
index d7e962f..c6d02cb 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorFactory.java
@@ -205,9 +205,7 @@ public class SentenceDetectorFactory extends BaseToolFactory {
}
private String eosCharArrayToString(char[] eosCharacters) {
- StringBuilder eosString = new StringBuilder();
- eosString.append(eosCharacters);
- return eosString.toString();
+ return String.valueOf(eosCharacters);
}
private char[] eosStringToCharArray(String eosCharacters) {
[42/50] [abbrv] opennlp git commit: OPENNLP-900: LemmatizerME and
DictionaryLemmatizer share data format
Posted by jo...@apache.org.
OPENNLP-900: LemmatizerME and DictionaryLemmatizer share data format
This closes #16
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/71e40d0b
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/71e40d0b
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/71e40d0b
Branch: refs/heads/889
Commit: 71e40d0b30fbef976da3595fef8736a4e1245466
Parents: 927ee0f
Author: Daniel Russ <dr...@mail.nih.gov>
Authored: Thu Dec 22 10:55:04 2016 -0500
Committer: Kottmann <jo...@apache.org>
Committed: Thu Dec 22 17:22:34 2016 +0100
----------------------------------------------------------------------
.../main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/71e40d0b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
index dde2d08..683b97c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
@@ -54,7 +54,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
try {
while ((line = breader.readLine()) != null) {
final String[] elems = line.split("\t");
- this.dictMap.put(Arrays.asList(elems[0], elems[2]), elems[1]);
+ this.dictMap.put(Arrays.asList(elems[0], elems[1]), elems[2]);
}
} catch (final IOException e) {
e.printStackTrace();
@@ -84,7 +84,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
keys.addAll(Arrays.asList(word.toLowerCase(), postag));
return keys;
}
-
+
public String[] lemmatize(final String[] tokens, final String[] postags) {
List<String> lemmas = new ArrayList<>();
for (int i = 0; i < tokens.length; i++) {
[08/50] [abbrv] opennlp git commit: Remove deprecated methods from
Doccat
Posted by jo...@apache.org.
Remove deprecated methods from Doccat
See issue OPENNLP-885
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/bbc5a34a
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/bbc5a34a
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/bbc5a34a
Branch: refs/heads/889
Commit: bbc5a34a8968dc39abcd24bdbc50b69851c6caa5
Parents: 0806562
Author: J�rn Kottmann <jo...@apache.org>
Authored: Thu Dec 15 17:52:26 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Thu Dec 15 17:52:26 2016 +0100
----------------------------------------------------------------------
.../java/opennlp/tools/doccat/DoccatModel.java | 26 -------------
.../tools/doccat/DocumentCategorizerME.java | 40 --------------------
.../opennlp/tools/doccat/DoccatFactoryTest.java | 2 +-
.../tools/doccat/DocumentCategorizerMETest.java | 2 +-
.../tools/doccat/DocumentCategorizerNBTest.java | 2 +-
.../uima/doccat/DocumentCategorizerTrainer.java | 11 +++++-
6 files changed, 12 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbc5a34a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
index e8c59fd..2bbaace 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DoccatModel.java
@@ -45,25 +45,6 @@ public class DoccatModel extends BaseModel {
checkArtifactMap();
}
- /**
- * @deprecated Use
- * {@link #DoccatModel(String, MaxentModel, Map, DoccatFactory)}
- * instead and pass in a {@link DoccatFactory}
- */
- protected DoccatModel(String languageCode, MaxentModel doccatModel,
- Map<String, String> manifestInfoEntries) {
- this(languageCode, doccatModel, manifestInfoEntries, new DoccatFactory());
- }
-
- /**
- * @deprecated Use
- * {@link #DoccatModel(String, MaxentModel, Map, DoccatFactory)}
- * instead and pass in a {@link DoccatFactory}
- */
- public DoccatModel(String languageCode, MaxentModel doccatModel) {
- this(languageCode, doccatModel, null);
- }
-
public DoccatModel(InputStream in) throws IOException, InvalidFormatException {
super(COMPONENT_NAME, in);
}
@@ -94,13 +75,6 @@ public class DoccatModel extends BaseModel {
return DoccatFactory.class;
}
- /**
- * @deprecated Use {@link #getMaxentModel()} instead.
- */
- public MaxentModel getChunkerModel() {
- return (MaxentModel) artifactMap.get(DOCCAT_MODEL_ENTRY_NAME);
- }
-
public MaxentModel getMaxentModel() {
return (MaxentModel) artifactMap.get(DOCCAT_MODEL_ENTRY_NAME);
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbc5a34a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
index 6ba4e08..2c31345 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/doccat/DocumentCategorizerME.java
@@ -176,30 +176,6 @@ public class DocumentCategorizerME implements DocumentCategorizer {
return model.getMaxentModel().getAllOutcomes(results);
}
- /**
- * @deprecated Use
- * {@link #train(String, ObjectStream, TrainingParameters, DoccatFactory)}
- * instead.
- */
- public static DoccatModel train(String languageCode, ObjectStream<DocumentSample> samples,
- TrainingParameters mlParams, FeatureGenerator... featureGenerators)
- throws IOException {
-
- if (featureGenerators.length == 0) {
- featureGenerators = new FeatureGenerator[]{defaultFeatureGenerator};
- }
-
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
-
- EventTrainer trainer = TrainerFactory.getEventTrainer(
- mlParams.getSettings(), manifestInfoEntries);
-
- MaxentModel model = trainer.train(
- new DocumentCategorizerEventStream(samples, featureGenerators));
-
- return new DoccatModel(languageCode, model, manifestInfoEntries);
- }
-
public static DoccatModel train(String languageCode, ObjectStream<DocumentSample> samples,
TrainingParameters mlParams, DoccatFactory factory)
throws IOException {
@@ -214,20 +190,4 @@ public class DocumentCategorizerME implements DocumentCategorizer {
return new DoccatModel(languageCode, model, manifestInfoEntries, factory);
}
-
- /**
- * Trains a doccat model with default feature generation.
- *
- * @param languageCode the language code
- * @param samples the samples
- * @return the trained doccat model
- * @throws IOException
- * @throws ObjectStreamException
- * @deprecated Use
- * {@link #train(String, ObjectStream, TrainingParameters, DoccatFactory)}
- * instead.
- */
- public static DoccatModel train(String languageCode, ObjectStream<DocumentSample> samples) throws IOException {
- return train(languageCode, samples, ModelUtil.createDefaultTrainingParameters(), defaultFeatureGenerator);
- }
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbc5a34a/opennlp-tools/src/test/java/opennlp/tools/doccat/DoccatFactoryTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/doccat/DoccatFactoryTest.java b/opennlp-tools/src/test/java/opennlp/tools/doccat/DoccatFactoryTest.java
index 786e708..8062762 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/doccat/DoccatFactoryTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/doccat/DoccatFactoryTest.java
@@ -49,7 +49,7 @@ public class DoccatFactoryTest {
private static DoccatModel train() throws IOException {
return DocumentCategorizerME.train("x-unspecified", createSampleStream(),
- TrainingParameters.defaultParams());
+ TrainingParameters.defaultParams(), new DoccatFactory());
}
private static DoccatModel train(DoccatFactory factory) throws IOException {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbc5a34a/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerMETest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerMETest.java
index 523772a..673198a 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerMETest.java
@@ -48,7 +48,7 @@ public class DocumentCategorizerMETest {
params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(0));
DoccatModel model = DocumentCategorizerME.train("x-unspecified", samples,
- params, new BagOfWordsFeatureGenerator());
+ params, new DoccatFactory());
DocumentCategorizer doccat = new DocumentCategorizerME(model);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbc5a34a/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerNBTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerNBTest.java b/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerNBTest.java
index a6e48de..67347de 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerNBTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/doccat/DocumentCategorizerNBTest.java
@@ -48,7 +48,7 @@ public class DocumentCategorizerNBTest {
params.put(AbstractTrainer.ALGORITHM_PARAM, NaiveBayesTrainer.NAIVE_BAYES_VALUE);
DoccatModel model = DocumentCategorizerME.train("x-unspecified", samples,
- params, new BagOfWordsFeatureGenerator());
+ params, new DoccatFactory());
DocumentCategorizer doccat = new DocumentCategorizerME(model);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/bbc5a34a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
----------------------------------------------------------------------
diff --git a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
index 1c266ad..31c6392 100644
--- a/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
+++ b/opennlp-uima/src/main/java/opennlp/uima/doccat/DocumentCategorizerTrainer.java
@@ -22,11 +22,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import opennlp.tools.doccat.DoccatFactory;
import opennlp.tools.doccat.DoccatModel;
import opennlp.tools.doccat.DocumentCategorizerME;
import opennlp.tools.doccat.DocumentSample;
import opennlp.tools.ml.maxent.GIS;
import opennlp.tools.util.ObjectStreamUtils;
+import opennlp.tools.util.TrainingParameters;
import opennlp.uima.util.CasConsumerUtil;
import opennlp.uima.util.OpennlpUtil;
import opennlp.uima.util.UimaUtil;
@@ -130,7 +132,12 @@ public class DocumentCategorizerTrainer extends CasConsumer_ImplBase {
GIS.PRINT_MESSAGES = false;
- DoccatModel categoryModel = DocumentCategorizerME.train(language, ObjectStreamUtils.createObjectStream(documentSamples));
+ TrainingParameters params = new TrainingParameters();
+ params.put(TrainingParameters.ITERATIONS_PARAM, Integer.toString(100));
+ params.put(TrainingParameters.CUTOFF_PARAM, Integer.toString(0));
+
+ DoccatModel categoryModel = DocumentCategorizerME.train(language,
+ ObjectStreamUtils.createObjectStream(documentSamples), params, new DoccatFactory());
File modelFile = new File(getUimaContextAdmin().getResourceManager()
.getDataPath() + File.separatorChar + mModelName);
@@ -152,4 +159,4 @@ public class DocumentCategorizerTrainer extends CasConsumer_ImplBase {
// dereference to allow garbage collection
documentSamples = null;
}
-}
\ No newline at end of file
+}
[03/50] [abbrv] opennlp git commit: Remove deprecated methods from
the Chunker
Posted by jo...@apache.org.
Remove deprecated methods from the Chunker
See issue OPENNLP-884
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/c657cded
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/c657cded
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/c657cded
Branch: refs/heads/889
Commit: c657cdeda97c3138221e4032b175669bab81db79
Parents: c9f2943
Author: J�rn Kottmann <jo...@apache.org>
Authored: Thu Nov 10 01:23:09 2016 +0100
Committer: J�rn Kottmann <jo...@apache.org>
Committed: Tue Nov 22 01:29:42 2016 +0100
----------------------------------------------------------------------
.../java/opennlp/tools/chunker/Chunker.java | 28 ----------
.../tools/chunker/ChunkerCrossValidator.java | 11 ----
.../tools/chunker/ChunkerEventStream.java | 10 ----
.../java/opennlp/tools/chunker/ChunkerME.java | 58 +-------------------
.../opennlp/tools/chunker/ChunkerModel.java | 41 --------------
.../opennlp/tools/chunker/ChunkerMETest.java | 23 --------
6 files changed, 2 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c657cded/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
index 02ccec0..b26f01b 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/Chunker.java
@@ -17,8 +17,6 @@
package opennlp.tools.chunker;
-import java.util.List;
-
import opennlp.tools.util.Sequence;
import opennlp.tools.util.Span;
@@ -28,19 +26,6 @@ import opennlp.tools.util.Span;
public interface Chunker {
/**
- * Generates chunk tags for the given sequence returning the result in a list.
- *
- * @param toks a list of the tokens or words of the sequence.
- * @param tags a list of the pos tags of the sequence.
- *
- * @return a list of chunk tags for each token in the sequence.
- *
- * @deprecated please use {@link #chunk(String[], String[])} instead.
- */
- @Deprecated
- public List<String> chunk(List<String> toks, List<String> tags);
-
- /**
* Generates chunk tags for the given sequence returning the result in an array.
*
* @param toks an array of the tokens or words of the sequence.
@@ -66,19 +51,6 @@ public interface Chunker {
* @param tags The pos-tags for the specified sentence.
*
* @return the top k chunk sequences for the specified sentence.
- *
- * @deprecated please use {@link #topKSequences(String[], String[])} instead.
- */
- @Deprecated
- public Sequence[] topKSequences(List<String> sentence, List<String> tags);
-
-
- /**
- * Returns the top k chunk sequences for the specified sentence with the specified pos-tags
- * @param sentence The tokens of the sentence.
- * @param tags The pos-tags for the specified sentence.
- *
- * @return the top k chunk sequences for the specified sentence.
*/
public Sequence[] topKSequences(String[] sentence, String[] tags);
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c657cded/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
index f264753..e9ce18d 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
@@ -34,17 +34,6 @@ public class ChunkerCrossValidator {
private ChunkerEvaluationMonitor[] listeners;
private ChunkerFactory chunkerFactory;
- /**
- * @deprecated Use {@link #ChunkerCrossValidator(String, TrainingParameters, ChunkerFactory, ChunkerEvaluationMonitor...)} instead.
- */
- public ChunkerCrossValidator(String languageCode, TrainingParameters params,
- ChunkerEvaluationMonitor... listeners) {
-
- this.languageCode = languageCode;
- this.params = params;
- this.listeners = listeners;
- }
-
public ChunkerCrossValidator(String languageCode, TrainingParameters params,
ChunkerFactory factory, ChunkerEvaluationMonitor... listeners) {
this.chunkerFactory = factory;
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c657cded/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
index 33efcdb..118357c 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEventStream.java
@@ -43,16 +43,6 @@ public class ChunkerEventStream extends AbstractEventStream<ChunkSample> {
this.cg = cg;
}
- /**
- * Creates a new event stream based on the specified data stream.
- * @param d The data stream for this event stream.
- *
- * @deprecated Use {@link #ChunkerEventStream(ObjectStream, ChunkerContextGenerator)} instead.
- */
- public ChunkerEventStream(ObjectStream<ChunkSample> d) {
- this(d, new DefaultChunkerContextGenerator());
- }
-
@Override
protected Iterator<Event> createEvents(ChunkSample sample) {
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c657cded/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
index 11d1c3b..e5b1073 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerME.java
@@ -67,7 +67,7 @@ public class ChunkerME implements Chunker {
* and use the {@link ChunkerFactory} to configure the {@link SequenceValidator} and {@link ChunkerContextGenerator}.
*/
@Deprecated
- public ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<String> sequenceValidator,
+ private ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<String> sequenceValidator,
ChunkerContextGenerator contextGenerator) {
this.sequenceValidator = sequenceValidator;
@@ -88,30 +88,11 @@ public class ChunkerME implements Chunker {
*
* @param model The model for this chunker.
* @param beamSize The size of the beam that should be used when decoding sequences.
- * @param sequenceValidator The {@link SequenceValidator} to determines whether the outcome
- * is valid for the preceding sequence. This can be used to implement constraints
- * on what sequences are valid.
- * @deprecated Use {@link #ChunkerME(ChunkerModel, int)} instead
- * and use the {@link ChunkerFactory} to configure the {@link SequenceValidator}.
- */
- @Deprecated
- public ChunkerME(ChunkerModel model, int beamSize,
- SequenceValidator<String> sequenceValidator) {
- this(model, beamSize, sequenceValidator,
- new DefaultChunkerContextGenerator());
- }
-
- /**
- * Initializes the current instance with the specified model and
- * the specified beam size.
- *
- * @param model The model for this chunker.
- * @param beamSize The size of the beam that should be used when decoding sequences.
*
* @deprecated beam size is now stored inside the model
*/
@Deprecated
- public ChunkerME(ChunkerModel model, int beamSize) {
+ private ChunkerME(ChunkerModel model, int beamSize) {
contextGenerator = model.getFactory().getContextGenerator();
sequenceValidator = model.getFactory().getSequenceValidator();
@@ -135,14 +116,6 @@ public class ChunkerME implements Chunker {
this(model, DEFAULT_BEAM_SIZE);
}
- @Deprecated
- public List<String> chunk(List<String> toks, List<String> tags) {
- bestSequence =
- model.bestSequence(toks.toArray(new String[toks.size()]), new Object[] { tags.toArray(new String[tags.size()]) },
- contextGenerator, sequenceValidator);
- return bestSequence.getOutcomes();
- }
-
public String[] chunk(String[] toks, String[] tags) {
bestSequence = model.bestSequence(toks, new Object[] {tags}, contextGenerator, sequenceValidator);
List<String> c = bestSequence.getOutcomes();
@@ -154,12 +127,6 @@ public class ChunkerME implements Chunker {
return ChunkSample.phrasesAsSpanList(toks, tags, preds);
}
- @Deprecated
- public Sequence[] topKSequences(List<String> sentence, List<String> tags) {
- return topKSequences(sentence.toArray(new String[sentence.size()]),
- tags.toArray(new String[tags.size()]));
- }
-
public Sequence[] topKSequences(String[] sentence, String[] tags) {
return model.bestSequences(DEFAULT_BEAM_SIZE, sentence,
new Object[] { tags }, contextGenerator, sequenceValidator);
@@ -235,25 +202,4 @@ public class ChunkerME implements Chunker {
return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory);
}
}
-
- /**
- * @deprecated Use
- * {@link ChunkerME#train(String, ObjectStream, TrainingParameters, ChunkerFactory)}
- * instead.
- */
- public static ChunkerModel train(String lang, ObjectStream<ChunkSample> in,
- ChunkerContextGenerator contextGenerator, TrainingParameters mlParams)
- throws IOException {
-
- Map<String, String> manifestInfoEntries = new HashMap<String, String>();
-
- ObjectStream<Event> es = new ChunkerEventStream(in, contextGenerator);
-
- EventTrainer trainer = TrainerFactory.getEventTrainer(
- mlParams.getSettings(), manifestInfoEntries);
-
- MaxentModel maxentModel = trainer.train(es);
-
- return new ChunkerModel(lang, maxentModel, manifestInfoEntries);
- }
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c657cded/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
index c677cd5..ed13f65 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerModel.java
@@ -19,9 +19,6 @@
package opennlp.tools.chunker;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
@@ -30,8 +27,6 @@ import java.util.Properties;
import opennlp.tools.ml.BeamSearch;
import opennlp.tools.ml.model.AbstractModel;
-import opennlp.tools.ml.model.BinaryFileDataReader;
-import opennlp.tools.ml.model.GenericModelReader;
import opennlp.tools.ml.model.MaxentModel;
import opennlp.tools.ml.model.SequenceClassificationModel;
import opennlp.tools.util.BaseToolFactory;
@@ -49,15 +44,6 @@ public class ChunkerModel extends BaseModel {
private static final String COMPONENT_NAME = "ChunkerME";
private static final String CHUNKER_MODEL_ENTRY_NAME = "chunker.model";
- /**
- * @deprecated Use
- * {@link #ChunkerModel(String, MaxentModel, Map, ChunkerFactory)}
- * instead.
- */
- public ChunkerModel(String languageCode, MaxentModel chunkerModel, Map<String, String> manifestInfoEntries) {
- this(languageCode, chunkerModel, ChunkerME.DEFAULT_BEAM_SIZE, manifestInfoEntries, new ChunkerFactory());
- }
-
public ChunkerModel(String languageCode, SequenceClassificationModel<String> chunkerModel,
Map<String, String> manifestInfoEntries, ChunkerFactory factory) {
super(COMPONENT_NAME, languageCode, manifestInfoEntries, factory);
@@ -81,15 +67,6 @@ public class ChunkerModel extends BaseModel {
checkArtifactMap();
}
- /**
- * @deprecated Use
- * {@link #ChunkerModel(String, MaxentModel, ChunkerFactory)
- * instead.}
- */
- public ChunkerModel(String languageCode, MaxentModel chunkerModel) {
- this(languageCode, chunkerModel, null, new ChunkerFactory());
- }
-
public ChunkerModel(String languageCode, MaxentModel chunkerModel, ChunkerFactory factory) {
this(languageCode, chunkerModel, null, factory);
}
@@ -159,22 +136,4 @@ public class ChunkerModel extends BaseModel {
public ChunkerFactory getFactory() {
return (ChunkerFactory) this.toolFactory;
}
-
- public static void main(String[] args) throws FileNotFoundException, IOException {
-
- if (args.length != 4){
- System.err.println("ChunkerModel -lang code packageName modelName");
- System.exit(1);
- }
-
- String lang = args[1];
- String packageName = args[2];
- String modelName = args[3];
-
- AbstractModel chunkerModel = new GenericModelReader(
- new BinaryFileDataReader(new FileInputStream(modelName))).getModel();
-
- ChunkerModel packageModel = new ChunkerModel(lang, chunkerModel);
- packageModel.serialize(new FileOutputStream(packageName));
- }
}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/c657cded/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
index bfb6b2c..3f37554 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerMETest.java
@@ -116,29 +116,6 @@ public class ChunkerMETest {
}
@Test
- public void testChunkAsList() throws Exception {
-
- @SuppressWarnings("deprecation")
- List<String> preds = chunker.chunk(Arrays.asList(toks1),
- Arrays.asList(tags1));
-
- assertEquals(Arrays.asList(expect1), preds);
- }
-
- @Test
- public void testTokenProbList() throws Exception {
-
- @SuppressWarnings("deprecation")
- Sequence[] preds = chunker.topKSequences(Arrays.asList(toks1),
- Arrays.asList(tags1));
-
- assertTrue(preds.length > 0);
- assertEquals(expect1.length, preds[0].getProbs().length);
- assertEquals(Arrays.asList(expect1), preds[0].getOutcomes());
- assertNotSame(Arrays.asList(expect1), preds[1].getOutcomes());
- }
-
- @Test
public void testTokenProbArray() throws Exception {
Sequence[] preds = chunker.topKSequences(toks1, tags1);