You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ko...@apache.org on 2018/06/27 13:25:07 UTC
[opennlp] branch master updated: OPENNLP-1197: fix bug,
recognizing symbols (including #, %. & etc) (#322)
This is an automated email from the ASF dual-hosted git repository.
koji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/master by this push:
new 8414c1e OPENNLP-1197: fix bug, recognizing symbols (including #, %. & etc) (#322)
8414c1e is described below
commit 8414c1e46cfa4aea6f20e0795b7fb487f72c0ac9
Author: Koji Sekiguchi <ko...@rondhuit.com>
AuthorDate: Wed Jun 27 22:25:02 2018 +0900
OPENNLP-1197: fix bug, recognizing symbols (including #, %. & etc) (#322)
---
.../src/main/java/opennlp/tools/util/featuregen/StringPattern.java | 3 +--
.../java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java | 3 +++
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java
index 458912f..af4cb96 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/util/featuregen/StringPattern.java
@@ -132,9 +132,8 @@ public class StringPattern {
}
}
else {
- if (ch == ',' || ch == '.' || ch == '?' || ch == '!') {
+ if (ch != '・' && ch != 'ー' && ch != '〜')
pattern &= ~(ALL_HIRAGANA | ALL_KATAKANA);
- }
}
}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java
index 7d7f233..5cde8db 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/featuregen/FeatureGeneratorUtilTest.java
@@ -47,6 +47,9 @@ public class FeatureGeneratorUtilTest {
Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("."));
Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("?"));
Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("!"));
+ Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("#"));
+ Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("%"));
+ Assert.assertEquals("other", FeatureGeneratorUtil.tokenFeature("&"));
}
@Test