You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/15 14:58:44 UTC
incubator-joshua git commit: updated OOVList test
Repository: incubator-joshua
Updated Branches:
refs/heads/7 bc050e3b5 -> eba33ba43
updated OOVList test
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/eba33ba4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/eba33ba4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/eba33ba4
Branch: refs/heads/7
Commit: eba33ba4333db06b94b2b7ae18aa6a3d758957ee
Parents: bc050e3
Author: Matt Post <po...@cs.jhu.edu>
Authored: Thu Sep 15 16:58:39 2016 +0200
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Sep 15 16:58:39 2016 +0200
----------------------------------------------------------------------
.../joshua/decoder/ff/tm/GrammarReader.java | 20 ++------------------
.../tm/hash_based/MemoryBasedBatchGrammar.java | 11 ++++++++---
.../resources/decoder/oov-list/joshua.config | 4 ++--
.../test/resources/decoder/oov-list/output.gold | 6 +++---
4 files changed, 15 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/eba33ba4/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
index 7e3d6cd..70e786c 100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
@@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory;
* @author Juri Ganitkevitch
*
*/
-public abstract class GrammarReader<R extends Rule> implements Iterable<R>, Iterator<R> {
+public abstract class GrammarReader<R extends Rule> implements Iterable<R>, Iterator<R>, AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(GrammarReader.class);
@@ -95,6 +95,7 @@ public abstract class GrammarReader<R extends Rule> implements Iterable<R>, Iter
throw new UnsupportedOperationException();
}
+ @Override
public void close() {
if (null != this.reader) {
try {
@@ -106,23 +107,6 @@ public abstract class GrammarReader<R extends Rule> implements Iterable<R>, Iter
}
}
- /**
- * For correct behavior <code>close</code> must be called on every GrammarReader, however this
- * code attempts to avoid resource leaks.
- *
- * @see org.apache.joshua.util.io.LineReader
- */
- @Override
- protected void finalize() throws Throwable {
- if (this.reader != null) {
- LOG.error("Grammar file stream was not closed, this indicates a coding error: {}",
- this.fileName);
- }
-
- this.close();
- super.finalize();
- }
-
@Override
public boolean hasNext() {
return lookAhead != null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/eba33ba4/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
index d5ef019..92566da 100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
@@ -21,6 +21,7 @@ package org.apache.joshua.decoder.ff.tm.hash_based;
import static org.apache.joshua.decoder.ff.tm.GrammarReader.createReader;
import java.io.IOException;
+import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -31,6 +32,7 @@ import org.apache.joshua.decoder.JoshuaConfiguration.OOVItem;
import org.apache.joshua.decoder.ff.FeatureFunction;
import org.apache.joshua.decoder.ff.FeatureVector;
import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+import org.apache.joshua.decoder.ff.tm.GrammarReader;
import org.apache.joshua.decoder.ff.tm.Rule;
import org.apache.joshua.decoder.ff.tm.Trie;
import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
@@ -83,10 +85,13 @@ public class MemoryBasedBatchGrammar extends AbstractGrammar {
this.grammarFile = grammarFile;
// ==== loading grammar
- for (Rule rule : createReader(formatKeyword, grammarFile, getOwner()))
- if (rule != null) {
- addRule(rule);
+ try(GrammarReader<Rule> reader = createReader(formatKeyword, grammarFile, getOwner());) {
+ for (Rule rule : reader) {
+ if (rule != null) {
+ addRule(rule);
+ }
}
+ }
this.printGrammar();
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/eba33ba4/joshua-core/src/test/resources/decoder/oov-list/joshua.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/oov-list/joshua.config b/joshua-core/src/test/resources/decoder/oov-list/joshua.config
index 8809206..883a023 100644
--- a/joshua-core/src/test/resources/decoder/oov-list/joshua.config
+++ b/joshua-core/src/test/resources/decoder/oov-list/joshua.config
@@ -25,7 +25,7 @@ feature-function = OOVPenalty
lm_0 1.2373676802179452
-tm_phrase_0 1
-tm_glue_0 1
+phrase_0 -1
+glue_0 -1
WordPenalty -3.6942747832593694
OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/eba33ba4/joshua-core/src/test/resources/decoder/oov-list/output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/decoder/oov-list/output.gold b/joshua-core/src/test/resources/decoder/oov-list/output.gold
index ee44a51..7cebb7b 100644
--- a/joshua-core/src/test/resources/decoder/oov-list/output.gold
+++ b/joshua-core/src/test/resources/decoder/oov-list/output.gold
@@ -1,3 +1,3 @@
-Goats eat cheese ||| tm_phrase_0=0.000 tm_glue_0=1.000 lm_0=-16.587 WordPenalty=-2.171 OOVPenalty=0.000 ||| -11.503
-i will go home ||| tm_phrase_0=0.000 tm_glue_0=1.000 lm_0=-12.155 WordPenalty=-2.606 OOVPenalty=0.000 ||| -4.414
-goets_OOV eet_OOV cheez_OOV ||| tm_phrase_0=0.000 tm_glue_0=1.000 lm_0=-17.700 WordPenalty=-2.171 OOVPenalty=-7.749 ||| -20.629
+Goats eat cheese ||| phrase_0=0.000000 lm_0=-16.587435 glue_0=-1.000000 WordPenalty=-2.171473 ||| -11.503
+i will go home ||| phrase_0=0.000000 lm_0=-12.155182 glue_0=-1.000000 WordPenalty=-2.605767 ||| -4.414
+goets_OOV eet_OOV cheez_OOV ||| phrase_0=0.000000 lm_0=-17.699974 glue_0=-1.000000 WordPenalty=-2.171473 OOVPenalty=-7.749294 ||| -20.629