You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/11/17 18:29:07 UTC
[3/9] incubator-joshua git commit: changed logic of glue grammars and
custom grammar
changed logic of glue grammars and custom grammar
Previously, custom grammars were always created. Now, they are only instantiated if a grammar with the owner "custom" is found in the config file. A suitable warning is thrown with a helpful error message if the user tries to add a rule when no custom grammar was loaded.
A glue grammar is created inline if no grammar owned by "glue" was found. Previously, one was created if there were no grammars period. This new method is more robust. It will be nice in version 7 to get rid of the separate PhraseTable, consolidating that and MemoryBasedBatchGrammar. Phrase-based decoding doesn't have to worry about there being glue rules present since the glue rules for phrase-based are hard-coded and part of the search algorithm, instead of just being treated like generic rules (like in CKY).
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/ef07c532
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/ef07c532
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/ef07c532
Branch: refs/heads/master
Commit: ef07c5326f737248ad494f401dff35d3414e0f56
Parents: 2229d1b
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed Nov 16 21:00:36 2016 -0500
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Nov 17 13:24:07 2016 -0500
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 104 ++++++++++---------
1 file changed, 56 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef07c532/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index ca2bfaa..84824a5 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -435,64 +435,72 @@ public class Decoder {
*/
private void initializeTranslationGrammars() throws IOException {
- if (joshuaConfiguration.tms.size() > 0) {
-
- // collect packedGrammars to check if they use a shared vocabulary
- final List<PackedGrammar> packed_grammars = new ArrayList<>();
-
- // tm = {thrax/hiero,packed,samt,moses} OWNER LIMIT FILE
- for (String tmLine : joshuaConfiguration.tms) {
-
- String type = tmLine.substring(0, tmLine.indexOf(' '));
- String[] args = tmLine.substring(tmLine.indexOf(' ')).trim().split("\\s+");
- HashMap<String, String> parsedArgs = FeatureFunction.parseArgs(args);
-
- String owner = parsedArgs.get("owner");
- int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
- String path = parsedArgs.get("path");
-
- Grammar grammar;
- if (! type.equals("moses") && ! type.equals("phrase")) {
- if (new File(path).isDirectory()) {
- try {
- PackedGrammar packed_grammar = new PackedGrammar(path, span_limit, owner, type, joshuaConfiguration);
- packed_grammars.add(packed_grammar);
- grammar = packed_grammar;
- } catch (FileNotFoundException e) {
- String msg = String.format("Couldn't load packed grammar from '%s'", path)
- + "Perhaps it doesn't exist, or it may be an old packed file format.";
- throw new RuntimeException(msg);
- }
- } else {
- // thrax, hiero, samt
- grammar = new MemoryBasedBatchGrammar(type, path, owner,
- joshuaConfiguration.default_non_terminal, span_limit, joshuaConfiguration);
- }
+ // collect packedGrammars to check if they use a shared vocabulary
+ final List<PackedGrammar> packed_grammars = new ArrayList<>();
+
+ // record the glue grammar so we can make sure there is one
+ Grammar glueGrammar = null;
- } else {
+ // tm = {thrax/hiero,packed,samt,moses} OWNER LIMIT FILE
+ for (String tmLine : joshuaConfiguration.tms) {
+
+ String type = tmLine.substring(0, tmLine.indexOf(' '));
+ String[] args = tmLine.substring(tmLine.indexOf(' ')).trim().split("\\s+");
+ HashMap<String, String> parsedArgs = FeatureFunction.parseArgs(args);
- joshuaConfiguration.search_algorithm = "stack";
- grammar = new PhraseTable(path, owner, type, joshuaConfiguration);
+ String owner = parsedArgs.get("owner");
+ int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
+ String path = parsedArgs.get("path");
+
+ Grammar grammar;
+ if (type.equals("moses") && ! type.equals("phrase")) {
+ joshuaConfiguration.search_algorithm = "stack";
+ grammar = new PhraseTable(path, owner, type, joshuaConfiguration);
+
+ } else {
+ if (new File(path).isDirectory()) {
+ try {
+ PackedGrammar packed_grammar = new PackedGrammar(path, span_limit, owner, type, joshuaConfiguration);
+ packed_grammars.add(packed_grammar);
+ grammar = packed_grammar;
+ } catch (FileNotFoundException e) {
+ String msg = String.format("Couldn't load packed grammar from '%s'", path)
+ + "Perhaps it doesn't exist, or it may be an old packed file format.";
+ throw new RuntimeException(msg);
+ }
+ } else {
+ // thrax, hiero, samt
+ grammar = new MemoryBasedBatchGrammar(type, path, owner,
+ joshuaConfiguration.default_non_terminal, span_limit, joshuaConfiguration);
}
+ }
+
+ this.grammars.add(grammar);
- this.grammars.add(grammar);
+ /* Record whether we saw a custom grammar for adding phrase entries */
+ if (getOwner(grammar.getOwner()).equals("custom")) {
+ this.customPhraseTable = grammar;
+ } else if (getOwner(grammar.getOwner()).equals("glue")) {
+ glueGrammar = grammar;
}
+ }
- checkSharedVocabularyChecksumsForPackedGrammars(packed_grammars);
+ checkSharedVocabularyChecksumsForPackedGrammars(packed_grammars);
- } else {
- LOG.warn("no grammars supplied! Supplying dummy glue grammar.");
- MemoryBasedBatchGrammar glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration, -1);
- glueGrammar.addGlueRules(featureFunctions);
+ /* Create a glue grammar if none was provided */
+ if (joshuaConfiguration.search_algorithm.equals("cky") && glueGrammar == null) {
+ LOG.warn("No glue grammar found! Creating dummy glue grammar.");
+ glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration, -1);
+ ((MemoryBasedBatchGrammar)glueGrammar).addGlueRules(featureFunctions);
this.grammars.add(glueGrammar);
}
+
+ /* Create a custom phrase table if none was found in the config file */
+ if (customPhraseTable == null) {
+ this.customPhraseTable = new PhraseTable("custom.grammar", "custom", "phrase", joshuaConfiguration);
+ this.grammars.add(this.customPhraseTable);
+ }
- /* Add the grammar for custom entries */
- if (joshuaConfiguration.search_algorithm.equals("stack"))
- this.customPhraseTable = new PhraseTable(null, "custom", "phrase", joshuaConfiguration);
- else
- this.customPhraseTable = new MemoryBasedBatchGrammar("custom", joshuaConfiguration, 20);
- this.grammars.add(this.customPhraseTable);
/* Create an epsilon-deleting grammar */
if (joshuaConfiguration.lattice_decoding) {