You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2024/11/12 20:51:17 UTC

(opennlp-models) branch OPENNLP-1640-Add-training-and-evaluation-of-Lemmatizer-models created (now d429c7a)

This is an automated email from the ASF dual-hosted git repository.

mawiesne pushed a change to branch OPENNLP-1640-Add-training-and-evaluation-of-Lemmatizer-models
in repository https://gitbox.apache.org/repos/asf/opennlp-models.git


      at d429c7a  OPENNLP-1640 Add training and evaluation of Lemmatizer models - adds execution phase for train & eval of lemma models for all supported languages - removes non-required line breaks in echo messages

This branch includes the following new commits:

     new d429c7a  OPENNLP-1640 Add training and evaluation of Lemmatizer models - adds execution phase for train & eval of lemma models for all supported languages - removes non-required line breaks in echo messages

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(opennlp-models) 01/01: OPENNLP-1640 Add training and evaluation of Lemmatizer models - adds execution phase for train & eval of lemma models for all supported languages - removes non-required line breaks in echo messages

Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mawiesne pushed a commit to branch OPENNLP-1640-Add-training-and-evaluation-of-Lemmatizer-models
in repository https://gitbox.apache.org/repos/asf/opennlp-models.git

commit d429c7a239eddea5246a4c54113df3c3b12bff6c
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Tue Nov 12 21:50:57 2024 +0100

    OPENNLP-1640 Add training and evaluation of Lemmatizer models
    - adds execution phase for train & eval of lemma models for all supported languages
    - removes non-required line breaks in echo messages
---
 .../src/main/resources/ud-train.sh                 | 41 ++++++++++++++++------
 1 file changed, 30 insertions(+), 11 deletions(-)

diff --git a/opennlp-models-training/opennlp-models-training-ud/src/main/resources/ud-train.sh b/opennlp-models-training/opennlp-models-training-ud/src/main/resources/ud-train.sh
index 516eb14..867e250 100755
--- a/opennlp-models-training/opennlp-models-training-ud/src/main/resources/ud-train.sh
+++ b/opennlp-models-training/opennlp-models-training-ud/src/main/resources/ud-train.sh
@@ -33,7 +33,7 @@ OPENNLP_CONFIG="ud-train.conf"
 # The directory a stable OpenNLP release is located in
 OPENNLP_HOME="./apache-opennlp-2.5.0"
 # The target version for training opennlp-models
-OPENNLP_MODEL_VERSION="1.1"
+OPENNLP_MODEL_VERSION="1.2"
 # The version of OpenNLP tools to use for training
 OPENNLP_VERSION_NUMERIC="2.5.0"
 # The directory the resulting binary models are written to
@@ -48,6 +48,7 @@ UD_HOME="./ud-treebanks-v2.14"
 TRAIN_TOKENIZER="true"
 TRAIN_POSTAGGER="true"
 TRAIN_SENTDETECT="true"
+TRAIN_LEMMATIZER="true"
 # If 'true', each resulting model is evaluated, 'false' otherwise
 EVAL_AFTER_TRAINING="true"
 # If 'true, training of experimental languages will be attempted, otherwise only stable languages & treebanks are used
@@ -65,7 +66,7 @@ mkdir -p ${OUTPUT_MODELS}
 
 for i in "${MODELS[@]}"
 do
-
+  echo -e "\n"
   echo $i
   LANG=`echo $i | cut -d'|' -f1`
   LANGCODE=`echo $i | cut -d'|' -f2`
@@ -74,15 +75,15 @@ do
 
   # Tokenizer model
   if [ ${TRAIN_TOKENIZER} == "true" ]; then
-    echo -e "\nTraining tokenizer model ${SUBSET} ${LANG}..."
+    echo -e "Training tokenizer model ${SUBSET} ${LANG}..."
     ${OPENNLP_HOME}/bin/opennlp TokenizerTrainer.conllu -params ${TRAIN_HOME}/${OPENNLP_CONFIG} -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -lang ${LANGCODE} -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-train.conllu -encoding ${ENCODING} > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.train
 
     if [ ${EVAL_AFTER_TRAINING} == "true" ]; then
-      echo -e "\nEvaluating tokenizer model ${SUBSET} ${LANG}..."
+      echo -e "Evaluating tokenizer model ${SUBSET} ${LANG}..."
       ${OPENNLP_HOME}/bin/opennlp TokenizerMEEvaluator.conllu -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-test.conllu -encoding ${ENCODING} > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.eval
     fi
     if [ ${CREATE_RELEASE} == "true" ]; then
-      echo -e "\nCreating hashes and ASC signature for tokenizer model ${SUBSET} ${LANG}..."
+      echo -e "Creating hashes and ASC signature for tokenizer model ${SUBSET} ${LANG}..."
       sha512sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha512
       sha256sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha256
       gpg --default-key $GPG_PUBLIC_KEY --armor --output ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.asc --detach-sign ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-tokens-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin
@@ -91,16 +92,16 @@ do
 
   # Sentence model
   if [ ${TRAIN_SENTDETECT} == "true" ]; then
-    echo -e "\nTraining sentence model ${SUBSET} ${LANG}..."
+    echo -e "Training sentence model ${SUBSET} ${LANG}..."
     ${OPENNLP_HOME}/bin/opennlp SentenceDetectorTrainer.conllu -params ${TRAIN_HOME}/${OPENNLP_CONFIG} -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -lang ${LANGCODE} -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-train.conllu -encoding ${ENCODING} -sentencesPerSample 10 > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.train
 
     if [ ${EVAL_AFTER_TRAINING} == "true" ]; then
-      echo -e "\nEvaluating sentence model ${SUBSET} ${LANG}..."
+      echo -e "Evaluating sentence model ${SUBSET} ${LANG}..."
       ${OPENNLP_HOME}/bin/opennlp SentenceDetectorEvaluator.conllu -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-test.conllu -encoding ${ENCODING} -sentencesPerSample 10 > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.eval
     fi
 
     if [ ${CREATE_RELEASE} == "true" ]; then
-      echo -e "\nCreating hashes and ASC signature for sentence model ${SUBSET} ${LANG}..."
+      echo -e "Creating hashes and ASC signature for sentence model ${SUBSET} ${LANG}..."
       sha512sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha512
       sha256sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha256
       gpg --default-key $GPG_PUBLIC_KEY --armor --output ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.asc --detach-sign ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-sentence-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin
@@ -109,22 +110,40 @@ do
 
   # POS model
   if [ ${TRAIN_POSTAGGER} == "true" ]; then
-    echo -e "\nTraining POS model ${SUBSET} ${LANG}..."
+    echo -e "Training POS model ${SUBSET} ${LANG}..."
     ${OPENNLP_HOME}/bin/opennlp POSTaggerTrainer.conllu -params ${TRAIN_HOME}/${OPENNLP_CONFIG} -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-train.conllu -encoding ${ENCODING} -lang ${LANGCODE} > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.train
 
     if [ ${EVAL_AFTER_TRAINING} == "true" ]; then
-      echo -e "\nEvaluating POS model ${SUBSET} ${LANG}..."
+      echo -e "Evaluating POS model ${SUBSET} ${LANG}..."
       ${OPENNLP_HOME}/bin/opennlp POSTaggerEvaluator.conllu -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-test.conllu -encoding ${ENCODING} > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.eval
     fi
 
     if [ ${CREATE_RELEASE} == "true" ]; then
-      echo -e "\nCreating hashes and ASC signature for POS model ${SUBSET} ${LANG}..."
+      echo -e "Creating hashes and ASC signature for POS model ${SUBSET} ${LANG}..."
       sha512sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha512
       sha256sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha256
       gpg --default-key $GPG_PUBLIC_KEY --armor --output ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.asc --detach-sign ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin
     fi
   fi
 
+  # Lemmatizer model
+  if [ ${TRAIN_LEMMATIZER} == "true" ]; then
+    echo -e "Training Lemmatizer model ${SUBSET} ${LANG}..."
+    ${OPENNLP_HOME}/bin/opennlp LemmatizerTrainerME.conllu -params ${TRAIN_HOME}/${OPENNLP_CONFIG} -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-train.conllu -encoding ${ENCODING} -lang ${LANGCODE} > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.train
+
+    if [ ${EVAL_AFTER_TRAINING} == "true" ]; then
+      echo -e "Evaluating Lemmatizer model ${SUBSET} ${LANG}..."
+      ${OPENNLP_HOME}/bin/opennlp LemmatizerEvaluator.conllu -model ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin -data ${UD_HOME}/UD_${LANG}-${SUBSET}/${LANGCODE}_${SUBSETLC}-ud-test.conllu -encoding ${ENCODING} > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.eval
+    fi
+
+    if [ ${CREATE_RELEASE} == "true" ]; then
+      echo -e "Creating hashes and ASC signature for Lemmatizer model ${SUBSET} ${LANG}..."
+      sha512sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha512
+      sha256sum ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin > ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.sha256
+      gpg --default-key $GPG_KEY --armor --output ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-pos-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin.asc --detach-sign ${OUTPUT_MODELS}/opennlp-${LANGCODE}-ud-${SUBSETLC}-lemmas-${OPENNLP_MODEL_VERSION}-${OPENNLP_VERSION_NUMERIC}.bin
+    fi
+  fi
+
 done
 
 # Conducts finalization steps to collect all training (and evaluation) log files into a zip