You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/09/20 18:16:13 UTC

(opennlp) branch OPENNLP-547-dependency-parser updated (5b05ca9b4 -> e2d7be80f)

This is an automated email from the ASF dual-hosted git repository.

krickert pushed a change to branch OPENNLP-547-dependency-parser
in repository https://gitbox.apache.org/repos/asf/opennlp.git


    omit 5b05ca9b4 Merge Apache main into OPENNLP-547-dependency-parser
    omit 12d9db03f Merge Apache main into OPENNLP-547-dependency-parser
    omit ebf96ee89 OPENNLP-547: Score with double precision so cached and direct parsing agree
    omit afd68fb1f OPENNLP-547: Keep parser internals package-private and validate input in the module
    omit 0e6511f92 OPENNLP-547: Test projectivity before deriving oracle transitions
    omit 1f4ed1b19 OPENNLP-547: Report format faults as InvalidFormatException
    omit 48ec28b3e OPENNLP-547: Expect InvalidFormatException for malformed models and word lines
    omit 0893833ee OPENNLP-547: Document what the dependency parser evaluation covers
    omit 205359e1e OPENNLP-547: Evaluate the dependency parsers across four UD 2.0 treebanks
    omit f24dd1547 OPENNLP-547: Back every manual example of the dependency parser with a test
    omit 279da7bf0 OPENNLP-547: Parameterize the dependency parser tests and name their fixtures
    omit 1ba04efd1 OPENNLP-547: Complete Javadoc, validation, and constants in the dependency parser
    omit 5b380e5f9 OPENNLP-547: Add a cross validator for dependency parsers
    omit 7712bf332 OPENNLP-547: Report attachment scores without punctuation
    omit 14f029ef8 OPENNLP-547: Evaluate both parsers on the UD English development split
    omit 549338a7e OPENNLP-547: Keep reserved embedding entries when training data spells them
    omit 40186281b OPENNLP-547: Skip CoNLL-U trees with a placeholder relation
    omit fb55c0fea OPENNLP-547: Show that reserved symbols in training data break reloading
    omit 9715fff9c OPENNLP-547: Show that an underscore relation is read as a label
    omit 266b6d419 Merge Apache main through Hunspell follow-up #1266 into OPENNLP-547-dependency-parser
    omit d6e74c4f0 Merge current main into dependency-parser
    omit 0763c2136 OPENNLP-547: Reconcile published parser history
    omit 5cc405820 OPENNLP-547: Document the remaining constructors and private contracts
    omit 7b121daf0 OPENNLP-547: Expose dependency evaluation metrics
    omit 4b6ed9f5d OPENNLP-547: State dependency evaluation inputs
    omit 5d36303f5 OPENNLP-547: Use toolkit case mapping
    omit 9e85bbe8e OPENNLP-547: Apply Unicode sigma context
    omit 95b887526 OPENNLP-547: Cover punctuation in sigma context
    omit 51ff6e831 OPENNLP-547: Cover uncased sigma context
    omit 01151a281 OPENNLP-547: Cover case-ignorable sigma context
    omit 365bfaea7 OPENNLP-547: Detect numerical training overflow
    omit 55092c8dd OPENNLP-547: Cover non-finite training weights
    omit c6a216f37 OPENNLP-547: Validate MaxEnt action inventories
    omit 590fc82de OPENNLP-547: Cover duplicate MaxEnt actions
    omit fc184aa38 OPENNLP-547: Cover incomplete MaxEnt actions
    omit 64c00b036 OPENNLP-547: Add parser examples and trim commentary
    omit 98488954d OPENNLP-547: Pin dependency evaluation data
    omit 17cc3ba39 OPENNLP-547: Cover treebank revision pinning
    omit d9accea89 OPENNLP-547: Exercise shared parser instances concurrently
    omit 4c9ddafe0 OPENNLP-547: Bound trainable model dimensions
    omit 8d2d0a785 OPENNLP-547: Cover unserializable trainer dimensions
    omit 2398ebcf2 OPENNLP-547: Normalize ordinary leading-star words
    omit 080298ae8 OPENNLP-547: Cover leading-star word normalization
    omit 3c2acf51a OPENNLP-547: Reject incomplete action inventories
    omit b39f10c50 OPENNLP-547: Cover incomplete action inventories
    omit 76498c1ab OPENNLP-547: Verify oracle and thread safety contracts
    omit b9752c442 OPENNLP-547: Train only observed parser actions
    omit 1a3491d5a OPENNLP-547: Cover feedforward action inventory
    omit 741c32037 OPENNLP-547: Validate model scores and CoNLL-U framing
    omit 3f71242da OPENNLP-547: Cover model scores and CoNLL-U framing
    omit e93a172d6 OPENNLP-547: Validate feedforward model files
    omit f3244b019 OPENNLP-547: Cover malformed feedforward models
    omit 428ecd467 OPENNLP-547: Enforce parser integrity boundaries
    omit 9351382f9 OPENNLP-547: Cover parser integrity boundaries
    omit 0330d00f8 OPENNLP-547: Parse CoNLL-U fields without regex
    omit d47da1aab depparse: Address review: cite specs and document private helpers
    omit f6f64100d depparse: Address review: decode outcomes once, fold duplication, name constants
    omit aa35faa00 dependency: Add a dependency parser manual chapter with a mirror-tested example
    omit 9a6c561a1 depparse: Validate relation labels through StringUtil.isBlank
    omit 4b10da1f1 depparse: Cache hidden-layer contributions per feature pair on frozen models
    omit 1daa2ac3e depparse: Restore the final-sigma rule in vocabulary normalization and make model bytes reproducible
    omit 7215d66d5 depparse: Refine into a new model, fail loud on unknown transitions, and read blanks and case through the project seams
    omit 300befc02 depparse: Document treebank acquisition for the gated evaluation with a download helper
    omit d1a98c64b depparse: State, transition, usage, and edge-case tests; expanded trainer and parser javadoc
    omit 5790610ea depparse: Global refinement with beam search and early update
    omit 2e731822a depparse: Beam search decoding for the feedforward parser
    omit 79c5773b8 depparse: Pretrained word embedding seeding for the feedforward trainer
    omit 0f499c1b6 depparse: Pure-Java feedforward neural tier, training and inference in the JVM
    omit 960b65b8d depparse: Valency and child features for the transition classifier
    omit 203b96bca depparse: Read CoNLL-U raw so multiword-token sentences keep their dependencies
    omit 8e026780c depparse: DependencyModel persistence and a gated Universal Dependencies evaluation
    omit fc54edc11 depparse: Arc-standard dependency parser skeleton with CoNLL-U reader and UAS/LAS evaluator
    omit 71da20136 OPENNLP-547: Merge updated document container branch
    omit e37f9c108 OPENNLP-1888: List the chunker and parser adapters in the manual
    omit 1b24daddd OPENNLP-1888: Add a ParserAnnotator providing a phrases layer
    omit 65c215b69 OPENNLP-1888: Add a ChunkerAnnotator providing a chunks layer
    omit 4d8ec0290 OPENNLP-547: Reconcile branch after document API cascade
    omit 84f059e4d OPENNLP-547: Parse CoNLL-U fields without regex
    omit 62a412dc3 depparse: Address review: cite specs and document private helpers
    omit e15e35e47 depparse: Address review: decode outcomes once, fold duplication, name constants
    omit d894cfc66 dependency: Add a dependency parser manual chapter with a mirror-tested example
    omit 9eb6c1b6c depparse: Validate relation labels through StringUtil.isBlank
    omit c8584a265 depparse: Cache hidden-layer contributions per feature pair on frozen models
    omit e83f03cb5 depparse: Restore the final-sigma rule in vocabulary normalization and make model bytes reproducible
    omit e48a5c948 depparse: Refine into a new model, fail loud on unknown transitions, and read blanks and case through the project seams
    omit a70ae3840 depparse: Document treebank acquisition for the gated evaluation with a download helper
    omit d1334ad27 depparse: State, transition, usage, and edge-case tests; expanded trainer and parser javadoc
    omit 2be280921 depparse: Global refinement with beam search and early update
    omit b0a3ed6b3 depparse: Beam search decoding for the feedforward parser
    omit 70987c15d depparse: Pretrained word embedding seeding for the feedforward trainer
    omit 282e886a5 depparse: Pure-Java feedforward neural tier, training and inference in the JVM
    omit dc070db0e depparse: Valency and child features for the transition classifier
    omit 1d8c366e2 depparse: Read CoNLL-U raw so multiword-token sentences keep their dependencies
    omit f5c11dc46 depparse: DependencyModel persistence and a gated Universal Dependencies evaluation
    omit 5115aaa40 depparse: Arc-standard dependency parser skeleton with CoNLL-U reader and UAS/LAS evaluator
    omit 22780da3d OPENNLP-1888: Place annotators with their components
    omit e380352c3 OPENNLP-1888: Move concrete annotators to runtime
    omit e69d29820 OPENNLP-547: Parse CoNLL-U fields without regex
    omit 89825abbc depparse: Address review: cite specs and document private helpers
    omit ef853a5bb depparse: Address review: decode outcomes once, fold duplication, name constants
    omit 0e13326f0 dependency: Add a dependency parser manual chapter with a mirror-tested example
    omit 299b16f5c depparse: Validate relation labels through StringUtil.isBlank
    omit 03a005dbc depparse: Cache hidden-layer contributions per feature pair on frozen models
    omit 09e82da86 depparse: Restore the final-sigma rule in vocabulary normalization and make model bytes reproducible
    omit d1072b78e depparse: Refine into a new model, fail loud on unknown transitions, and read blanks and case through the project seams
    omit 752c05c5c depparse: Document treebank acquisition for the gated evaluation with a download helper
    omit 273499018 depparse: State, transition, usage, and edge-case tests; expanded trainer and parser javadoc
    omit 63cd289fc depparse: Global refinement with beam search and early update
    omit ecfdc9eff depparse: Beam search decoding for the feedforward parser
    omit 944cff4d0 depparse: Pretrained word embedding seeding for the feedforward trainer
    omit cf9080ba8 depparse: Pure-Java feedforward neural tier, training and inference in the JVM
    omit 14a226a25 depparse: Valency and child features for the transition classifier
    omit f7182f63d depparse: Read CoNLL-U raw so multiword-token sentences keep their dependencies
    omit c828b8666 depparse: DependencyModel persistence and a gated Universal Dependencies evaluation
    omit ffb99d009 depparse: Arc-standard dependency parser skeleton with CoNLL-U reader and UAS/LAS evaluator
    omit f4dd343a9 OPENNLP-1888: Name adapters by their simple class name
    omit 550bc4490 OPENNLP-1888: Pin that adapters name themselves by class name
    omit 0705c5d73 OPENNLP-1888: State span offsets in UTF-16 units in the manual
    omit 09cd47d59 OPENNLP-1888: Merge with a single map copy in ImmutableDocument
    omit 562af8714 OPENNLP-1888: Pin merge semantics across Document implementations
    omit 75fcac098 OPENNLP-1888: Pin what layer equality compares for KEEP_EQUAL
    omit 844128fad OPENNLP-1888: Name the differing contents when KEEP_EQUAL rejects a layer
    omit 4f5c0cacd OPENNLP-1888: Pin a differing-contents message for KEEP_EQUAL rejection
    omit 3245eff1c OPENNLP-1888: Render the layer figure monospaced and tighten the prose
    omit 1297f725d OPENNLP-1888: Add a duplicate-layer policy to Document.merge
    omit adb15b426 OPENNLP-1888: Pin the duplicate-layer policy contract for merge
    omit aa5aebae7 OPENNLP-1888: Document the parallel fan-out join in the manual
    omit 919cf590e OPENNLP-1888: Implement Document.merge as a default method
    omit 9442b91b6 OPENNLP-1888: Pin the Document.merge contract
    omit 608430990 OPENNLP-1888: Rework the document chapter after docs review
    omit 689636105 OPENNLP-1888: Name the null argument, avoid per-call key set wrapping, pin blank and span edge cases
    omit e9062250b OPENNLP-1888: Address review: fold duplicate test fixtures and pin null rejection
    omit 15102d75d OPENNLP-1888: Address review: shared annotator helpers, hardened container contract
    omit 04532170c OPENNLP-1888: Anchor the Document thread-safety note to immutability
    omit d43c1ec9a OPENNLP-1888: Javadoc container overrides and cite the pipeline example test
    omit 266e780d9 OPENNLP-1888: Create toolkit layer keys through namespace-applying factories on Layers
    omit e4d6ed718 OPENNLP-1888: Document the gold-layer convention
    omit b39dd25d1 OPENNLP-1888: State the index-reference invariants in the specification text
    omit f8b2ee97e OPENNLP-1888: Declare per-key positional or document scope
    omit 1ba2fcdae OPENNLP-1888: Namespace the standard layer key ids
    omit eb277fbb6 OPENNLP-1888: Manual chapter, review-convention pass, and key placement rule
    omit 99e0b83fe OPENNLP-1888: Add StringUtil.isBlank following the toolkit whitespace definition
    omit 6b384b670 OPENNLP-1888: Lemmatize per sentence, validate adapter outputs, clear adaptive data on failure
    omit e9b53a7a6 OPENNLP-1888: Parse per sentence in the adapters, distinguish empty from missing layers, validate providers at build time
    omit e8a2ca47f OPENNLP-1888: Pipeline example and contract tests for the document container, javadoc precision pass
    omit 3e382b31b OPENNLP-1888: Lemma and stem layer adapters
    omit 49b9612df OPENNLP-1888: Document annotation container: typed offset-anchored layers over the original text
     add 5bcbc29ee OPENNLP-1950: Fix ADChunkSampleStream fails with NPE on the FlorestaVirgem corpus (#1299)
     add 229e4255a OPENNLP-1952: Report a CoNLL-U multiword token range without its word lines (#1301)
     add 0ad45ba21 OPENNLP-1951: Clear adaptive data at each new LIT or CIE text in ADNameSampleStream (#1300)
     add dff5e5d17 OPENNLP-1955: Reject surrogate separators, check split against String.split in the test, single scan in the report listener
     add c694a276b OPENNLP-1945: Publish snapshot artifacts for a branch on demand (#1307)
     add a5689758e OPENNLP-1957: Use Apache Commons Secure XML for JAXP factory creation (#1306)
     add 9f60bf305 OPENNLP-1972: Resolve service-loaded components through a generic provider SPI- #1308
     add d32d4d84e Minor: Regenerated NOTICE File for 9f60bf305798e2b07d5b13fc2207a594ce965624 (#1309)
     add c47eeafef OPENNLP-1973: Regex removal (prep): Add a CompatibilityMode switch for the earlier output (#1312)
     add 89efde8d7 OPENNLP-1983: Enable GitHub issues and discussions (#1316)
     add 6331d6e19 OPENNLP-1937: Expose text embeddings through the provider SPI (#1310)
     add 4cce2c59c OPENNLP-1928: Regex removal (1/10): Replace trivial patterns with character scans (#1311)
     add f2ab1ccd4 OPENNLP-1946: UrlCharSequenceNormalizer removes whole URLs including ports, userinfo and non-ASCII hosts (#1315)
     add 9e7bee3ad OPENNLP-1930: Regex removal (2/10): Parse Arvores Deitadas markup with cursor scans (#1276)
     add c36ff62f7 OPENNLP-1942: Test the shared deep-learning encoder and fix the token type javadoc (#1288)
     add e2d7be80f OPENNLP-547: Add the classical dependency parser component

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (5b05ca9b4)
            \
             N -- N -- N   refs/heads/OPENNLP-547-dependency-parser (e2d7be80f)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

No new revisions were added by this update.

Summary of changes:
 .asf.yaml                                          |    7 +
 .github/workflows/publish-snapshots.yml            |   69 +-
 .gitignore                                         |    3 +
 NOTICE                                             |    1 +
 SECURITY.md                                        |    8 +-
 .../opennlp/tools/depparse/DependencyGraph.java    |    7 +-
 .../opennlp/tools/depparse/DependencySample.java   |    7 +-
 .../DocumentAnnotatorProvider.java}                |   14 +-
 .../EmbeddingException.java}                       |   27 +-
 .../opennlp/tools/embeddings/TextEmbedder.java     |  108 ++
 .../TextEmbedderProvider.java}                     |   14 +-
 .../opennlp/tools/tokenize/WordpieceEncoder.java   |    3 +-
 .../java/opennlp/tools/util/CompatibilityMode.java |  134 +++
 .../main/java/opennlp/tools/util/StringUtil.java   |  181 ++-
 ...eption.java => AmbiguousProviderException.java} |   26 +-
 .../main/java/opennlp/tools/util/ext/Provider.java |   90 ++
 .../ext/ProviderResolutionException.java}          |   34 +-
 .../java/opennlp/tools/util/ext/ProviderSpec.java  |  423 +++++++
 .../java/opennlp/tools/util/ext/Providers.java     | 1066 +++++++++++++++++
 ...tion.java => UnsatisfiedProviderException.java} |   26 +-
 .../tools/depparse/DependencySampleTest.java       |   21 +
 .../opennlp/tools/embeddings/TextEmbedderTest.java |   82 ++
 .../opennlp/tools/util/CompatibilityModeTest.java  |  126 ++
 .../opennlp/tools/util/StringUtilSplitTest.java    |   78 +-
 .../opennlp/tools/util/ext/ProviderSpecTest.java   |  278 +++++
 .../java/opennlp/tools/util/ext/ProvidersTest.java | 1215 ++++++++++++++++++++
 .../src/main/java/opennlp/tools/cmdline/CLI.java   |   10 +
 .../tools/cmdline/FineGrainedReportListener.java   |   42 +-
 .../DependencyModelLoader.java}                    |   17 +-
 .../DependencyParserCrossValidatorTool.java        |   83 ++
 .../depparse/DependencyParserEvaluatorTool.java    |   70 ++
 .../cmdline/depparse/DependencyParserMETool.java   |   80 ++
 .../depparse/DependencyParserTrainerTool.java      |   73 ++
 .../{chunker => depparse}/TrainingParams.java      |    6 +-
 .../depparse/DependencyParserToolsTest.java        |  104 ++
 .../opennlp-formats/dev/README-ud-treebanks.md     |   73 --
 .../opennlp-formats/dev/download-ud-treebank.sh    |   87 --
 .../tools/cmdline/StreamFactoryRegistry.java       |    2 +
 .../formats/DetokenizerSampleStreamFactory.java    |    6 +-
 .../tools/formats/ad/ADChunkSampleStream.java      |   13 +-
 .../java/opennlp/tools/formats/ad/ADMetadata.java  |  151 +++
 .../tools/formats/ad/ADNameSampleStream.java       |  315 +++--
 .../tools/formats/ad/ADPOSSampleStream.java        |   33 +-
 .../tools/formats/ad/ADSentenceSampleStream.java   |   23 +-
 .../opennlp/tools/formats/ad/ADSentenceStream.java |  748 ++++++++++--
 .../conllu/ConlluDependencySampleStream.java       |   25 +-
 ...va => ConlluDependencySampleStreamFactory.java} |   36 +-
 .../opennlp/tools/formats/conllu/ConlluStream.java |   23 +-
 .../formats/nkjp/NKJPSegmentationDocument.java     |    4 +-
 .../tools/formats/nkjp/NKJPTextDocument.java       |    4 +-
 .../tools/formats/ad/ADChunkSampleStreamTest.java  |   70 ++
 .../opennlp/tools/formats/ad/ADMetadataTest.java   |  167 +++
 .../tools/formats/ad/ADNameSampleStreamTest.java   |  234 ++++
 .../tools/formats/ad/ADPOSSampleStreamTest.java    |   31 +
 .../tools/formats/ad/ADSentenceParserTest.java     |   74 ++
 .../formats/ad/ADSentenceSampleStreamTest.java     |   48 +
 .../tools/formats/ad/ADSentenceStreamTest.java     |  638 ++++++++++
 .../conllu/ConlluDependencyParserUsageTest.java    |   55 -
 .../conllu/ConlluDependencySampleStreamTest.java   |   17 +
 .../tools/formats/conllu/ConlluStreamTest.java     |  104 ++
 .../conllu/UdTreebankDownloadScriptTest.java       |   62 -
 .../src/main/java/opennlp/dl/Tokens.java           |    4 +-
 .../dl/vectors/OnnxTextEmbedderProvider.java       |  190 +++
 .../dl/{Tokens.java => vectors/Pooling.java}       |   18 +-
 .../java/opennlp/dl/vectors/SentenceVectorsDL.java |  355 +++++-
 .../opennlp.tools.embeddings.TextEmbedderProvider  |    8 +-
 .../opennlp/dl/AbstractDLEncodeTokensTest.java     |  354 ++++++
 .../test/java/opennlp/dl/CreateTokenizerTest.java  |   27 -
 .../src/test/java/opennlp/dl/ModelFreeDL.java      |   54 +
 .../dl/vectors/OnnxTextEmbedderProviderTest.java   |   93 ++
 .../dl/vectors/SentenceVectorsDLEmbedderTest.java  |  264 +++++
 .../opennlp/dl/vectors/gen_tiny_pooled_model.py    |   98 ++
 .../opennlp/dl/vectors/gen_tiny_vectors_model.py   |   57 +
 .../resources/opennlp/dl/vectors/tiny-pooled.onnx  |  Bin 0 -> 513 bytes
 .../resources/opennlp/dl/vectors/tiny-vectors.onnx |  Bin 0 -> 373 bytes
 opennlp-core/opennlp-runtime/pom.xml               |    5 +
 .../opennlp/tools/depparse/ArcStandardState.java   |   29 +-
 .../tools/depparse/DependencyContextGenerator.java |    8 +-
 .../tools/depparse/DependencyCrossValidator.java   |   13 +-
 .../opennlp/tools/depparse/DependencyModel.java    |   31 +-
 .../tools/depparse/DependencyParserFactory.java    |   60 +
 .../opennlp/tools/depparse/DependencyParserME.java |   25 +-
 .../opennlp/tools/depparse/FeedforwardContext.java |  102 --
 .../tools/depparse/FeedforwardDependencyModel.java |  951 ---------------
 .../depparse/FeedforwardDependencyParser.java      |  228 ----
 .../depparse/FeedforwardDependencyTrainer.java     | 1175 -------------------
 .../java/opennlp/tools/depparse/ParserInput.java   |   59 -
 .../java/opennlp/tools/depparse/Transition.java    |   16 +-
 .../serializer/DictionaryEntryPersistor.java       |    5 +-
 .../DefaultLemmatizerContextGenerator.java         |   10 +-
 .../namefind/BilouNameFinderSequenceValidator.java |    4 +-
 .../main/java/opennlp/tools/namefind/BioCodec.java |   41 +-
 .../java/opennlp/tools/namefind/NameFinderME.java  |   18 -
 .../namefind/NameFinderSequenceValidator.java      |    4 +-
 .../tools/postag/DefaultPOSContextGenerator.java   |    9 +-
 .../tools/stemmer/StemmerAnnotatorProvider.java    |   94 ++
 .../src/main/java/opennlp/tools/util/XmlUtil.java  |  177 ++-
 .../tools/util/featuregen/BrownCluster.java        |    6 +-
 .../tools/util/featuregen/GeneratorFactory.java    |    3 +-
 .../featuregen/TokenPatternFeatureGenerator.java   |   19 +-
 .../opennlp/tools/util/normalizer/AsciiChars.java  |    5 +
 .../tools/util/normalizer/MailAddressScan.java     |  107 ++
 .../util/normalizer/UrlCharSequenceNormalizer.java |  324 ++++--
 ...pennlp.tools.document.DocumentAnnotatorProvider |    8 +-
 .../depparse/DependencyParserEdgeCaseTest.java     |   95 +-
 .../depparse/DependencyParserFactoryTest.java      |   66 ++
 .../depparse/FeedforwardDependencyParserTest.java  |  746 ------------
 .../depparse/FeedforwardDependencyScoringTest.java |  456 --------
 .../depparse/TestDependencyParserFactory.java}     |   28 +-
 .../DefaultLemmatizerContextGeneratorTest.java     |   48 +
 .../java/opennlp/tools/namefind/BioCodecTest.java  |   34 +
 .../postag/DefaultPOSContextGeneratorTest.java     |   23 +
 .../stemmer/StemmerAnnotatorProviderTest.java      |  136 +++
 .../test/java/opennlp/tools/util/XmlUtilTest.java  |  172 ++-
 .../tools/util/featuregen/BrownClusterTest.java    |   92 ++
 .../TokenPatternFeatureGeneratorTest.java          |   18 +
 ...CharSequenceNormalizerCharacterizationTest.java |   17 +-
 opennlp-distr/src/main/readme/NOTICE               |    1 +
 opennlp-docs/src/docbkx/corpora.xml                |   73 ++
 opennlp-docs/src/docbkx/dependency.xml             |   70 +-
 opennlp-docs/src/docbkx/introduction.xml           |  151 +++
 opennlp-docs/src/docbkx/langdetect.xml             |    8 +-
 opennlp-docs/src/docbkx/machine-learning.xml       |   37 +
 .../opennlp/dl/vectors/SentenceVectorsDLEval.java  |   16 +-
 .../opennlp/tools/eval/ArvoresDeitadasEval.java    |    2 +-
 .../tools/eval/UniversalDependencyParserEval.java  |   56 +-
 .../src/test/java/opennlp/uima/util/CasUtil.java   |   18 +-
 .../java/opennlp/tools/util/StringUtilTest.java    |  178 +++
 .../normalizer/UrlCharSequenceNormalizerTest.java  |  302 ++++-
 pom.xml                                            |   29 +-
 rat-excludes                                       |    4 +
 131 files changed, 10530 insertions(+), 4980 deletions(-)
 copy opennlp-api/src/main/java/opennlp/tools/{sentiment/SentimentEvaluationMonitor.java => document/DocumentAnnotatorProvider.java} (68%)
 copy opennlp-api/src/main/java/opennlp/tools/{models/ClassPathLoaderException.java => embeddings/EmbeddingException.java} (63%)
 create mode 100644 opennlp-api/src/main/java/opennlp/tools/embeddings/TextEmbedder.java
 copy opennlp-api/src/main/java/opennlp/tools/{sentiment/SentimentEvaluationMonitor.java => embeddings/TextEmbedderProvider.java} (69%)
 create mode 100644 opennlp-api/src/main/java/opennlp/tools/util/CompatibilityMode.java
 copy opennlp-api/src/main/java/opennlp/tools/util/ext/{ExtensionNotLoadedException.java => AmbiguousProviderException.java} (63%)
 create mode 100644 opennlp-api/src/main/java/opennlp/tools/util/ext/Provider.java
 copy opennlp-api/src/main/java/opennlp/tools/{models/ClassPathLoaderException.java => util/ext/ProviderResolutionException.java} (55%)
 create mode 100644 opennlp-api/src/main/java/opennlp/tools/util/ext/ProviderSpec.java
 create mode 100644 opennlp-api/src/main/java/opennlp/tools/util/ext/Providers.java
 copy opennlp-api/src/main/java/opennlp/tools/util/ext/{ExtensionNotLoadedException.java => UnsatisfiedProviderException.java} (62%)
 create mode 100644 opennlp-api/src/test/java/opennlp/tools/embeddings/TextEmbedderTest.java
 create mode 100644 opennlp-api/src/test/java/opennlp/tools/util/CompatibilityModeTest.java
 create mode 100644 opennlp-api/src/test/java/opennlp/tools/util/ext/ProviderSpecTest.java
 create mode 100644 opennlp-api/src/test/java/opennlp/tools/util/ext/ProvidersTest.java
 copy opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/{lemmatizer/LemmatizerModelLoader.java => depparse/DependencyModelLoader.java} (72%)
 create mode 100644 opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserCrossValidatorTool.java
 create mode 100644 opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserEvaluatorTool.java
 create mode 100644 opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserMETool.java
 create mode 100644 opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/depparse/DependencyParserTrainerTool.java
 copy opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/{chunker => depparse}/TrainingParams.java (86%)
 create mode 100644 opennlp-core/opennlp-cli/src/test/java/opennlp/tools/cmdline/depparse/DependencyParserToolsTest.java
 delete mode 100644 opennlp-core/opennlp-formats/dev/README-ud-treebanks.md
 delete mode 100755 opennlp-core/opennlp-formats/dev/download-ud-treebank.sh
 create mode 100644 opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADMetadata.java
 copy opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/conllu/{ConlluPOSSampleStreamFactory.java => ConlluDependencySampleStreamFactory.java} (63%)
 create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADMetadataTest.java
 create mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/ad/ADSentenceParserTest.java
 delete mode 100644 opennlp-core/opennlp-formats/src/test/java/opennlp/tools/formats/conllu/UdTreebankDownloadScriptTest.java
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/vectors/OnnxTextEmbedderProvider.java
 copy opennlp-core/opennlp-ml/opennlp-dl/src/main/java/opennlp/dl/{Tokens.java => vectors/Pooling.java} (69%)
 copy opennlp-eval-tests/lang/ml/NaiveBayesTrainerParams.txt => opennlp-core/opennlp-ml/opennlp-dl/src/main/resources/META-INF/services/opennlp.tools.embeddings.TextEmbedderProvider (82%)
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/AbstractDLEncodeTokensTest.java
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/ModelFreeDL.java
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/OnnxTextEmbedderProviderTest.java
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/java/opennlp/dl/vectors/SentenceVectorsDLEmbedderTest.java
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/gen_tiny_pooled_model.py
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/gen_tiny_vectors_model.py
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/tiny-pooled.onnx
 create mode 100644 opennlp-core/opennlp-ml/opennlp-dl/src/test/resources/opennlp/dl/vectors/tiny-vectors.onnx
 create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/DependencyParserFactory.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardContext.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyModel.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyParser.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/FeedforwardDependencyTrainer.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/depparse/ParserInput.java
 create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/StemmerAnnotatorProvider.java
 create mode 100644 opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/MailAddressScan.java
 copy opennlp-eval-tests/lang/ml/NaiveBayesTrainerParams.txt => opennlp-core/opennlp-runtime/src/main/resources/META-INF/services/opennlp.tools.document.DocumentAnnotatorProvider (82%)
 create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/DependencyParserFactoryTest.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyParserTest.java
 delete mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/depparse/FeedforwardDependencyScoringTest.java
 copy opennlp-core/opennlp-runtime/src/{main/java/opennlp/tools/util/featuregen/DocumentBeginFeatureGeneratorFactory.java => test/java/opennlp/tools/depparse/TestDependencyParserFactory.java} (58%)
 create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/lemmatizer/DefaultLemmatizerContextGeneratorTest.java
 create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/stemmer/StemmerAnnotatorProviderTest.java
 create mode 100644 opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/featuregen/BrownClusterTest.java