You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2024/09/05 11:32:46 UTC
(opennlp) branch OPENNLP-1600-Ability-to-disable-POS-mapper created (now a1ee9fdb)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a change to branch OPENNLP-1600-Ability-to-disable-POS-mapper
in repository https://gitbox.apache.org/repos/asf/opennlp.git
at a1ee9fdb OPENNLP-1600 - Ability to disable POS mapper
This branch includes the following new commits:
new a1ee9fdb OPENNLP-1600 - Ability to disable POS mapper
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opennlp) 01/01: OPENNLP-1600 - Ability to disable POS mapper
Posted by rz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch OPENNLP-1600-Ability-to-disable-POS-mapper
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit a1ee9fdbd5cce10d1bd5ea8c96ac2229a9810881
Author: Richard Zowalla <rz...@apache.org>
AuthorDate: Thu Sep 5 13:32:36 2024 +0200
OPENNLP-1600 - Ability to disable POS mapper
---
.../java/opennlp/tools/postag/POSTagFormat.java | 2 +-
.../opennlp/tools/postag/POSTagFormatMapper.java | 26 +++++++++++++++++++++-
.../java/opennlp/tools/postag/POSTaggerME.java | 11 +++++----
.../java/opennlp/tools/postag/POSTaggerMETest.java | 16 +++++++++++++
4 files changed, 49 insertions(+), 6 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormat.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormat.java
index ddb9cc5f..66affc56 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormat.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormat.java
@@ -23,5 +23,5 @@ package opennlp.tools.postag;
*/
public enum POSTagFormat {
- UD, PENN, UNKNOWN
+ UD, PENN, CUSTOM, UNKNOWN
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormatMapper.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormatMapper.java
index b01a36ac..38d6e27f 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormatMapper.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTagFormatMapper.java
@@ -119,7 +119,7 @@ public class POSTagFormatMapper {
}
/**
- * Converts a a list of tags to the specified format.
+ * Converts a list of tags to the specified format.
*
* @param tags a list of tags to be converted.
* @return an array containing the converted tags with the same order and size as the given input list.
@@ -218,4 +218,28 @@ public class POSTagFormatMapper {
final POSTagFormatMapper mapper = new POSTagFormatMapper(posModel.getPosSequenceModel().getOutcomes());
return mapper.getGuessedFormat();
}
+
+ public static class NoOp extends POSTagFormatMapper {
+
+ protected NoOp() {
+ super(new String[0]);
+ }
+
+ @Override
+ public String[] convertTags(List<String> tags) {
+ Objects.requireNonNull(tags, "tags must not be NULL.");
+ return tags.toArray(new String[0]);
+ }
+
+ @Override
+ public String convertTag(String tag) {
+ return tag;
+ }
+
+ @Override
+ public POSTagFormat getGuessedFormat() {
+ return POSTagFormat.CUSTOM;
+ }
+
+ }
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
index 2deb5738..cd419e67 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerME.java
@@ -85,7 +85,8 @@ public class POSTaggerME implements POSTagger {
private final SequenceValidator<String> sequenceValidator;
private final POSTagFormat posTagFormat;
- private final POSTagFormatMapper posTagFormatMapper;
+
+ protected final POSTagFormatMapper posTagFormatMapper;
/**
* Initializes a {@link POSTaggerME} by downloading a default model for a given
@@ -151,7 +152,9 @@ public class POSTaggerME implements POSTagger {
this.model = new BeamSearch(beamSize, model.getArtifact(POSModel.POS_MODEL_ENTRY_NAME), 0);
}
- this.posTagFormatMapper = new POSTagFormatMapper(getAllPosTags());
+ this.posTagFormatMapper = (format == POSTagFormat.CUSTOM)
+ ? new POSTagFormatMapper.NoOp()
+ : new POSTagFormatMapper(getAllPosTags());
}
@@ -193,7 +196,8 @@ public class POSTaggerME implements POSTagger {
}
private String[] convertTags(List<String> t) {
- if (posTagFormatMapper.getGuessedFormat() == posTagFormat) {
+ if (posTagFormat == POSTagFormat.CUSTOM
+ || posTagFormatMapper.getGuessedFormat() == posTagFormat) {
return t.toArray(new String[0]);
} else {
return posTagFormatMapper.convertTags(t);
@@ -402,5 +406,4 @@ public class POSTaggerME implements POSTagger {
logger.info("... finished expanding POS Dictionary. [ {} ms]", (System.nanoTime() - start) / 1000000);
}
-
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
index d01fe3ab..49da2042 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/postag/POSTaggerMETest.java
@@ -93,6 +93,22 @@ public class POSTaggerMETest extends AbstractModelLoaderTest {
POSTagFormat.UD), sentence, expected);
}
+ @Test
+ void testPOSTaggerMappingNoOp() throws IOException {
+ final String[] sentence = {
+ "The",
+ "driver",
+ "got",
+ "badly",
+ "injured",
+ "."};
+
+ final String[] expected = {"DT", "NN", "VBD", "RB", "VBN", "."};
+ //leave it as is
+ testPOSTagger(new POSTaggerME(trainPennFormatPOSModel(ModelType.MAXENT),
+ POSTagFormat.CUSTOM), sentence, expected);
+ }
+
@Test
@EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
void testPOSTaggerDefault() throws IOException {