You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by rz...@apache.org on 2026/07/16 08:26:48 UTC
(opennlp) branch main updated: OPENNLP-244: Document the BioNLP converter (#1175)
This is an automated email from the ASF dual-hosted git repository.
rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/main by this push:
new b4f52fa02 OPENNLP-244: Document the BioNLP converter (#1175)
b4f52fa02 is described below
commit b4f52fa021c068d97b892454f27b2f9641112feb
Author: jerry317395616 <14...@users.noreply.github.com>
AuthorDate: Thu Jul 16 16:26:43 2026 +0800
OPENNLP-244: Document the BioNLP converter (#1175)
---
.../BioNLP2004NameSampleStreamFactoryTest.java | 24 +++++++
opennlp-docs/src/docbkx/corpora.xml | 74 ++++++++++++++++++++++
2 files changed, 98 insertions(+)
diff --git a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
index 25c19fa2c..2f0e45f1c 100644
--- a/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
+++ b/opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/BioNLP2004NameSampleStreamFactoryTest.java
@@ -17,7 +17,9 @@
package opennlp.tools.formats;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
@@ -32,6 +34,7 @@ import opennlp.tools.util.ObjectStream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class BioNLP2004NameSampleStreamFactoryTest extends
@@ -87,6 +90,27 @@ public class BioNLP2004NameSampleStreamFactoryTest extends
}
}
+ @Test
+ void testConvertManualExample() throws IOException {
+ String input = "IL-2\tB-DNA\n"
+ + "receptor\tI-DNA\n"
+ + "activates\tO\n"
+ + "T\tB-cell_type\n"
+ + "cells\tI-cell_type\n"
+ + ".\tO\n\n";
+ int types = BioNLP2004NameSampleStream.GENERATE_DNA_ENTITIES
+ | BioNLP2004NameSampleStream.GENERATE_CELLTYPE_ENTITIES;
+
+ try (ObjectStream<NameSample> stream = new BioNLP2004NameSampleStream(
+ () -> new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8)), types)) {
+ NameSample sample = stream.read();
+ assertNotNull(sample);
+ assertEquals("<START:DNA> IL-2 receptor <END> activates "
+ + "<START:cell_type> T cells <END> .", sample.toString());
+ assertNull(stream.read());
+ }
+ }
+
@Test
void testCreateWithUnsupportedTypes() throws IOException {
try (ObjectStream<NameSample> stream = factory.create(
diff --git a/opennlp-docs/src/docbkx/corpora.xml b/opennlp-docs/src/docbkx/corpora.xml
index db038f6f9..f42c9a6b6 100644
--- a/opennlp-docs/src/docbkx/corpora.xml
+++ b/opennlp-docs/src/docbkx/corpora.xml
@@ -522,6 +522,80 @@ path: /dev/opennlp/trunk/opennlp-tools/en-ontonotes.bin]]>
</section>
</section>
+ <section xml:id="tools.corpora.bionlp2004">
+ <title>BioNLP/NLPBA 2004</title>
+ <para>
+ The BioNLP/NLPBA 2004 shared task uses biomedical abstracts derived from the GENIA
+ corpus for named entity recognition. It defines five entity types:
+ <literal>DNA</literal>, <literal>RNA</literal>, <literal>protein</literal>,
+ <literal>cell_type</literal>, and <literal>cell_line</literal>. The
+ <link xlink:href="https://aclanthology.org/W04-1213/">shared task paper</link>
+ describes the corpus and annotation task. OpenNLP identifies this input format as
+ <literal>bionlp2004</literal>.
+ </para>
+
+ <section xml:id="tools.corpora.bionlp2004.getting">
+ <title>Getting the data</title>
+ <para>
+ A copy of the original training and evaluation files is available in the
+ <filename>original-data</filename> directory of the
+ <link xlink:href="https://github.com/spyysalo/jnlpba">JNLPBA corpus repository</link>.
+ The corpus is not distributed with OpenNLP. Review the terms supplied with the corpus
+ before using or redistributing it.
+ </para>
+ </section>
+
+ <section xml:id="tools.corpora.bionlp2004.format">
+ <title>Input format</title>
+ <para>
+ The converter reads UTF-8 text with one token and one IOB tag per line, separated by a
+ tab. A blank line ends a sentence. <literal>B-type</literal> starts an entity,
+ <literal>I-type</literal> continues it, and <literal>O</literal> marks a token outside
+ an entity. Lines beginning with <literal>###MEDLINE:</literal> mark document boundaries
+ in the original corpus. For example:
+ </para>
+ <screen>
+<![CDATA[IL-2 B-DNA
+receptor I-DNA
+activates O
+T B-cell_type
+cells I-cell_type
+. O
+
+]]>
+ </screen>
+ </section>
+
+ <section xml:id="tools.corpora.bionlp2004.converting">
+ <title>Converting the data</title>
+ <para>
+ Pass the desired entity types to <option>-types</option> as a comma-separated list. The
+ values are case-sensitive and must use the names shown above. This command selects all
+ five types and writes native OpenNLP name finder samples to
+ <filename>jnlpba.train</filename>:
+ </para>
+ <screen>
+<![CDATA[$ opennlp TokenNameFinderConverter bionlp2004 \
+ -data original-data/train/Genia4ERtask1.iob2 \
+ -types DNA,protein,cell_type,cell_line,RNA \
+ > jnlpba.train]]>
+ </screen>
+ <para>
+ The input example from the previous section is converted to:
+ </para>
+ <screen>
+<![CDATA[<START:DNA> IL-2 receptor <END> activates <START:cell_type> T cells <END> .]]>
+ </screen>
+ <para>
+ To convert only a subset, list only those types, for example
+ <literal>-types protein,cell_type</literal>. The resulting file can be passed to the
+ standard <command>TokenNameFinderTrainer</command>. See
+ <xref linkend="tools.namefind.training"/> for training
+ parameters and data recommendations.
+ </para>
+ </section>
+ </section>
+
<section xml:id="tools.corpora.brat">
<title>Brat Format Support</title>
<para>