You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2023/09/04 06:53:07 UTC
[opennlp] branch main updated: OPENNLP-1512 Fix incorrect encoding used in Conll02NameSampleStream
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new 8da0a97a OPENNLP-1512 Fix incorrect encoding used in Conll02NameSampleStream
8da0a97a is described below
commit 8da0a97a02d8ea251ded5edc50cb2182ebb94e51
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Fri Sep 1 18:36:23 2023 +0200
OPENNLP-1512 Fix incorrect encoding used in Conll02NameSampleStream
---
.../java/opennlp/tools/cmdline/AbstractConverterTool.java | 2 +-
.../opennlp/tools/formats/Conll02NameSampleStream.java | 15 ++++++++++++++-
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/AbstractConverterTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/AbstractConverterTool.java
index 6cd3fb9c..3d563c4e 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/AbstractConverterTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/AbstractConverterTool.java
@@ -113,7 +113,7 @@ public abstract class AbstractConverterTool<T,P> extends TypedCmdLineTool<T,P> {
}
try (ObjectStream<T> sampleStream = streamFactory.create(formatArgs)) {
- Object sample;
+ T sample;
while ((sample = sampleStream.read()) != null) {
logger.info(sample.toString());
}
diff --git a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
index 08317810..ef6de3b9 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/formats/Conll02NameSampleStream.java
@@ -91,7 +91,20 @@ public class Conll02NameSampleStream implements ObjectStream<NameSample> {
* @throws IOException Thrown if IO errors occurred.
*/
public Conll02NameSampleStream(LANGUAGE lang, InputStreamFactory in, int types) throws IOException {
- this (lang, new PlainTextByLineStream(in, StandardCharsets.UTF_8), types);
+ /*
+ * NOTE: KEEP this encoding here! The original CONLL 2002 data is provided as: ISO_8859_1.
+ */
+ this (lang, new PlainTextByLineStream(in, StandardCharsets.ISO_8859_1), types);
+ /*
+ * If related files are (incorrectly) interpreted as 'UTF-8' without prior conversion of
+ * the train/test files, then á, é, ñ,.. will be misinterpreted during processing and in
+ * resulting outcomes, e.g. produced via TokenNameFinderConverter.
+ *
+ * As a consequence, users of related tooling (OpenNLP Doc: CONLL 2002) will thus suffer
+ * from corrupted intermediate files, as an out-of the box experience.
+ *
+ * Details see: https://issues.apache.org/jira/browse/OPENNLP-1512
+ */
}
static Span extract(int begin, int end, String beginTag) throws InvalidFormatException {