You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/19 10:15:26 UTC
[05/21] incubator-joshua git commit: Fixed two of the three
NumTranslationOptionsTest test cases to work with new configuration. Packed
Grammar test fails, probably missing configuration parameter.
Fixed two of the three NumTranslationOptionsTest test cases to work with new configuration. Packed Grammar test fails, probably missing configuration parameter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d117c177
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d117c177
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d117c177
Branch: refs/heads/7_confsystem
Commit: d117c177d9130281f2b057fd1fdaf7de43c4f3b8
Parents: 89a112a
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 18:41:57 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 18:46:02 2016 +0200
----------------------------------------------------------------------
.../NumTranslationOptionsNoDotChartTest.conf | 28 +++++++
.../NumTranslationOptionsNoDotChartTest.gold | 4 +
.../cky/NumTranslationOptionsPackedTest.conf | 27 +++++++
.../cky/NumTranslationOptionsPackedTest.gold | 4 +
.../decoder/cky/NumTranslationOptionsTest.conf | 27 +++++++
.../decoder/cky/NumTranslationOptionsTest.gold | 4 +
.../decoder/cky/NumTranslationOptionsTest.in | 1 +
.../decoder/cky/NumTranslationOptionsTest.java | 83 ++++++--------------
8 files changed, 118 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf
new file mode 100644
index 0000000..a80bd4c
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf
@@ -0,0 +1,28 @@
+num_translation_options = 3
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c ||| %s ||| %f
+use_unique_nbest = true
+top_n = 5
+use_dot_chart = false
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/num_translation_options/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = 1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold
new file mode 100644
index 0000000..d130edd
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold
@@ -0,0 +1,4 @@
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
new file mode 100644
index 0000000..dc4d6a3
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
@@ -0,0 +1,27 @@
+num_translation_options = 3
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c ||| %s ||| %f
+use_unique_nbest = true
+top_n = 5
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/num_translation_options/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = PackedGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.packed }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = 1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold
new file mode 100644
index 0000000..d9e97f5
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold
@@ -0,0 +1,4 @@
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf
new file mode 100644
index 0000000..a8b8723
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf
@@ -0,0 +1,27 @@
+num_translation_options = 3
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c ||| %s ||| %f
+use_unique_nbest = true
+top_n = 5
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/num_translation_options/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = 1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold
new file mode 100644
index 0000000..d9e97f5
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold
@@ -0,0 +1,4 @@
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in
new file mode 100644
index 0000000..1ef14e5
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in
@@ -0,0 +1 @@
+yo quiero taco bell
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
index 7038241..66db0ad 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
@@ -18,23 +18,21 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
/**
* Tests that num_translation_options is enforced for hierarchical decoders
*/
public class NumTranslationOptionsTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -45,62 +43,27 @@ public class NumTranslationOptionsTest {
}
}
- @Test
- public void givenInput_whenDecodingWithNumTranslationOptions3_thenScoreAndTranslationCorrect()
- throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/input");
-
- // When
- configureDecoder("src/test/resources/decoder/num_translation_options/joshua.config", true);
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/output.gold");
- assertEquals(decodedStrings, goldStrings);
- }
-
- @Test
- public void givenInput_whenDecodingWithNumTranslationOptions3AndNoDotChart_thenScoreAndTranslationCorrect()
- throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/input");
-
- // When
- configureDecoder("src/test/resources/decoder/num_translation_options/joshua.config", false);
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold");
- assertEquals(decodedStrings, goldStrings);
+ @DataProvider(name = "testFiles")
+ public Object[][] lmFiles() {
+ return new Object[][] {
+ { "NumTranslationOptionsTest.conf", "NumTranslationOptionsTest.in",
+ "NumTranslationOptionsTest.gold" },
+ { "NumTranslationOptionsNoDotChartTest.conf", "NumTranslationOptionsTest.in",
+ "NumTranslationOptionsNoDotChartTest.gold" },
+ { "NumTranslationOptionsPackedTest.conf", "NumTranslationOptionsTest.in",
+ "NumTranslationOptionsPackedTest.gold" } };
}
- @Test
- public void givenInput_whenDecodingWithNumTranslationOptions3AndPacked_thenScoreAndTranslationCorrect()
- throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/input");
+ @Test(dataProvider = "testFiles")
+ public void givenInput_whenDecodingWithNumTranslationOptions_thenScoreAndTranslationCorrect(
+ String confFile, String inFile, String goldFile) throws Exception {
+ String inputPath = this.getClass().getResource(inFile).getFile();
+ String goldPath = this.getClass().getResource(goldFile).getFile();
+ Config config = parseResources(this.getClass(), confFile)
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- // When
- configureDecoder("src/test/resources/decoder/num_translation_options/joshua-packed.config",
- true);
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/output-packed.gold");
- assertEquals(decodedStrings, goldStrings);
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
- public void configureDecoder(String pathToConfig, boolean useDotChart) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- joshuaConfig.use_dot_chart = useDotChart;
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
}