You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/19 10:15:27 UTC
[06/21] incubator-joshua git commit: Fixed RescoringTest to work with
new configuration.
Fixed RescoringTest to work with new configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/a7e2adf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/a7e2adf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/a7e2adf4
Branch: refs/heads/7_confsystem
Commit: a7e2adf4faafbdf4b8ba75b8c67ecf62c62ab56a
Parents: d117c17
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 18:58:02 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 18:58:02 2016 +0200
----------------------------------------------------------------------
.../joshua/decoder/cky/RescoringTest.conf | 28 ++++++++++++++++
.../joshua/decoder/cky/RescoringTest.gold | 12 +++++++
.../apache/joshua/decoder/cky/RescoringTest.in | 2 ++
.../joshua/decoder/cky/RescoringTest.java | 35 ++++++--------------
4 files changed, 52 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf
new file mode 100644
index 0000000..8c8a751
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf
@@ -0,0 +1,28 @@
+rescore_forest = true
+rescore_forest_weight = 100
+mark_oovs = true
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %s ||| %f ||| %c
+use_unique_nbest = true
+top_n = 2
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/rescoring/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/rescoring/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/rescoring/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = -1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold
new file mode 100644
index 0000000..6ed6cd2
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold
@@ -0,0 +1,12 @@
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the kid eated the cockroach ||| pt_0=15.000000 lm_0=-20.053476 glue_0=-5.000000 WordPenalty=-3.040061 ||| -31.773
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in
new file mode 100644
index 0000000..5562a01
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in
@@ -0,0 +1,2 @@
+el nino tomo la cucaracha ||| ||| the boy ate the cockroach
+el nino tomo la cucaracha ||| ||| the big storm swarmed the coast ||| the big storm only swarmed the coast
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
index 7f4cdc0..de59d08 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
@@ -18,22 +18,17 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class RescoringTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -47,23 +42,13 @@ public class RescoringTest {
@Test
public void givenInput_whenDecodingWithRescoring_thenScoreAndTranslationCorrect()
throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/rescoring/input.txt");
+ String inputPath = this.getClass().getResource("RescoringTest.in").getFile();
+ String goldPath = this.getClass().getResource("RescoringTest.gold").getFile();
+ Config config = parseResources(this.getClass(), "RescoringTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- // When
- configureDecoder("src/test/resources/decoder/rescoring/joshua.config");
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/rescoring/output.gold");
- assertEquals(decodedStrings, goldStrings);
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
- public void configureDecoder(String pathToConfig) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
}