You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/19 10:15:22 UTC
[01/21] incubator-joshua git commit: Merge pull request #1 from
apache/7_confsystem
Repository: incubator-joshua
Updated Branches:
refs/heads/7_confsystem 8f47187e4 -> 1d4309ae1
Merge pull request #1 from apache/7_confsystem
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/7726e7d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/7726e7d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/7726e7d4
Branch: refs/heads/7_confsystem
Commit: 7726e7d4248556814e41d8dc4feba1f29d5c771c
Parents: 6f3e3c4 4cb838c
Author: Felix Hieber <fh...@users.noreply.github.com>
Authored: Sat Sep 17 19:06:01 2016 +0200
Committer: GitHub <no...@github.com>
Committed: Sat Sep 17 19:06:01 2016 +0200
----------------------------------------------------------------------
.../joshua/decoder/cky/BnEnBerkeleyLMTest.conf | 42 +
.../joshua/decoder/cky/BnEnBerkeleyLMTest.gold | 943 +++++++++++++++++++
.../joshua/decoder/cky/BnEnClassLMTest.conf | 44 +
.../joshua/decoder/cky/BnEnClassLMTest.gold | 887 +++++++++++++++++
.../joshua/decoder/cky/BnEnDecodingTest.java | 169 ++--
.../org/apache/joshua/decoder/cky/BnEnHiero.in | 100 ++
.../joshua/decoder/cky/BnEnHieroTest.conf | 43 +
.../joshua/decoder/cky/BnEnHieroTest.gold | 805 ++++++++++++++++
.../org/apache/joshua/decoder/cky/TestUtil.java | 118 ++-
scripts/compat/sevenize_my_conf_plz.py | 114 +++
10 files changed, 3101 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
[16/21] incubator-joshua git commit: Fixed TargetBigramTest to work
with new configuration. Compiles but does not pass yet.
Posted by mj...@apache.org.
Fixed TargetBigramTest to work with new configuration. Compiles but does not pass yet.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/14ede56c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/14ede56c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/14ede56c
Branch: refs/heads/7_confsystem
Commit: 14ede56c1bc51c4b92a02349c53daae735fc3ffb
Parents: 0ede005
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Mon Sep 19 10:10:20 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Mon Sep 19 10:10:20 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/TargetBigram.java | 37 +++++++++++---------
1 file changed, 21 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/14ede56c/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
index 30e6d1a..add96ef 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
@@ -18,14 +18,16 @@
*/
package org.apache.joshua.decoder.cky;
+import static com.typesafe.config.ConfigFactory.parseString;
import static org.apache.joshua.decoder.cky.TestUtil.translate;
import static org.testng.Assert.assertEquals;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class TargetBigram {
private static final String INPUT = "this is a test";
@@ -34,36 +36,39 @@ public class TargetBigram {
private static final String GOLD_THRESHOLD10 = "this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_UNK=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_is=1.000 TargetBigram_a_UNK=1.000 TargetBigram_is_a=1.000 ||| 0.000";
private static final String VOCAB_PATH = "src/test/resources/decoder/target-bigram/vocab";
+ private static final String CONF_TOPN2 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
+ + VOCAB_PATH + ", top-n = 2 } ]";
+ private static final String CONF_TOPN3_THRESHOLD20 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
+ + VOCAB_PATH + ", top-n = 3, threshold = 20 } ]";
+ private static final String CONF_THRESHOLD10 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
+ + VOCAB_PATH + ", threshold = 10 } ]";
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@Test
- public void givenInput_whenNotUsingSourceAnnotations_thenOutputCorrect() throws Exception {
- setUp("TargetBigram -vocab " + VOCAB_PATH + " -top-n 2");
- String output = translate(INPUT, decoder, joshuaConfig);
+ public void givenInput_whenDecodingWithTargetBigramAndTopN2_thenOutputCorrect() {
+ setUp(CONF_TOPN2);
+ String output = translate(INPUT, decoder);
assertEquals(output.trim(), GOLD_TOPN2);
}
@Test
- public void givenInput_whenUsingSourceAnnotations_thenOutputCorrect() throws Exception {
- setUp("TargetBigram -vocab " + VOCAB_PATH + " -top-n 3 -threshold 20");
- String output = translate(INPUT, decoder, joshuaConfig);
+ public void givenInput_whenDecodingWithTargetBigramAndTopN3Threshold20_thenOutputCorrect() {
+ setUp(CONF_TOPN3_THRESHOLD20);
+ String output = translate(INPUT, decoder);
assertEquals(output.trim(), GOLD_TOPN3_THRESHOLD20);
}
@Test
- public void givenInput_whenUsingSourceAnnotations_thenOutputCorrect2() throws Exception {
- setUp("TargetBigram -vocab " + VOCAB_PATH + " -threshold 10");
- String output = translate(INPUT, decoder, joshuaConfig);
+ public void givenInput_whenDecodingWithTargetBigramThreshold10_thenOutputCorrect2() {
+ setUp(CONF_THRESHOLD10);
+ String output = translate(INPUT, decoder);
assertEquals(output.trim(), GOLD_THRESHOLD10);
}
- public void setUp(String featureFunction) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.features.add(featureFunction);
- joshuaConfig.outputFormat = "%s ||| %f ||| %c";
- decoder = new Decoder(joshuaConfig);
+ public void setUp(String configuration) {
+ Config config = parseString(configuration).withFallback(Decoder.getDefaultFlags());
+ decoder = new Decoder(config);
}
@AfterMethod
[02/21] incubator-joshua git commit: Merge pull request #2 from
apache/7_confsystem
Posted by mj...@apache.org.
Merge pull request #2 from apache/7_confsystem
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/c23af309
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/c23af309
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/c23af309
Branch: refs/heads/7_confsystem
Commit: c23af3090b308a0b4a31794e44e3155b2c4613e6
Parents: 7726e7d 8f47187
Author: Felix Hieber <fh...@users.noreply.github.com>
Authored: Sun Sep 18 11:48:53 2016 +0200
Committer: GitHub <no...@github.com>
Committed: Sun Sep 18 11:48:53 2016 +0200
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/cky/BnEn.in | 100 ++
.../joshua/decoder/cky/BnEnBerkeleyLMTest.conf | 2 +-
.../joshua/decoder/cky/BnEnClassLMTest.conf | 2 +-
.../joshua/decoder/cky/BnEnDecodingTest.java | 15 +-
.../org/apache/joshua/decoder/cky/BnEnHiero.in | 100 --
.../joshua/decoder/cky/BnEnHieroTest.conf | 2 +-
.../joshua/decoder/cky/BnEnPackedTest.conf | 43 +
.../joshua/decoder/cky/BnEnPackedTest.gold | 864 +++++++++++++++++
.../apache/joshua/decoder/cky/BnEnSAMTTest.conf | 43 +
.../apache/joshua/decoder/cky/BnEnSAMTTest.gold | 864 +++++++++++++++++
.../joshua/decoder/cky/ConstrainedTest.java | 84 +-
.../joshua/decoder/cky/DenormalizationTest.java | 79 +-
.../joshua/decoder/cky/DoNotCrashTest.java | 89 +-
.../joshua/decoder/cky/NoGrammarTest.java | 75 +-
.../apache/joshua/decoder/cky/TooLongTest.java | 56 +-
.../resources/bn-en/hiero/class_lm_2gram.gz | Bin 18052 -> 0 bytes
.../src/test/resources/bn-en/hiero/input.bn | 100 --
.../bn-en/hiero/joshua-berkeleylm.config | 47 -
.../resources/bn-en/hiero/joshua-classlm.config | 52 -
.../test/resources/bn-en/hiero/joshua.config | 48 -
.../src/test/resources/bn-en/hiero/lm.gz | Bin 2466496 -> 0 bytes
.../bn-en/hiero/output-berkeleylm.gold | 943 -------------------
.../resources/bn-en/hiero/output-classlm.gold | 887 -----------------
.../src/test/resources/bn-en/hiero/output.gold | 805 ----------------
.../test/resources/bn-en/hiero/output.gold.bleu | 14 -
.../bn-en/hiero/output.scores.berkeleylm.gold | 100 --
.../resources/bn-en/hiero/output.scores.gold | 805 ----------------
.../resources/bn-en/hiero/test-berkeleylm.sh | 33 -
.../test/resources/bn-en/hiero/test-classlm.sh | 32 -
.../test/resources/bn-en/hiero/test-filter.sh | 35 -
.../src/test/resources/bn-en/hiero/test.sh | 35 -
.../src/test/resources/bn-en/hiero/topN.pl | 18 -
joshua-core/src/test/resources/bn-en/lm.gz | Bin 0 -> 2466496 bytes
.../src/test/resources/bn-en/packed/input.bn | 100 --
.../test/resources/bn-en/packed/joshua.config | 50 -
.../src/test/resources/bn-en/packed/lm.gz | Bin 2466496 -> 0 bytes
.../src/test/resources/bn-en/packed/output.gold | 862 -----------------
.../resources/bn-en/packed/output.scores.gold | 862 -----------------
.../test/resources/bn-en/packed/reference.en.0 | 100 --
.../test/resources/bn-en/packed/reference.en.1 | 100 --
.../test/resources/bn-en/packed/reference.en.2 | 100 --
.../test/resources/bn-en/packed/reference.en.3 | 100 --
.../resources/bn-en/packed/reference.en.all | 400 --------
.../src/test/resources/bn-en/packed/test.sh | 20 -
.../src/test/resources/bn-en/samt/input.bn | 100 --
.../src/test/resources/bn-en/samt/joshua.config | 51 -
joshua-core/src/test/resources/bn-en/samt/lm.gz | Bin 2466496 -> 0 bytes
.../src/test/resources/bn-en/samt/output.gold | 862 -----------------
.../test/resources/bn-en/samt/output.gold.bleu | 14 -
.../resources/bn-en/samt/output.scores.gold | 862 -----------------
.../test/resources/bn-en/samt/reference.en.0 | 100 --
.../test/resources/bn-en/samt/reference.en.1 | 100 --
.../test/resources/bn-en/samt/reference.en.2 | 100 --
.../test/resources/bn-en/samt/reference.en.3 | 100 --
.../src/test/resources/bn-en/samt/test.sh | 35 -
.../resources/decoder/constrained/gold.scores | 27 -
.../test/resources/decoder/constrained/input.bn | 8 -
.../resources/decoder/constrained/joshua.config | 45 -
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 30 -
.../test/resources/decoder/constrained/test.sh | 30 -
.../test/resources/decoder/constrained/weights | 22 -
.../resources/decoder/denormalization/input.txt | 1 -
.../decoder/denormalization/output.expected | 1 -
.../resources/decoder/denormalization/test.sh | 30 -
.../src/test/resources/decoder/dont-crash/input | 10 -
.../test/resources/decoder/dont-crash/test.sh | 29 -
.../resources/decoder/empty-test/.gitignore | 3 -
.../src/test/resources/decoder/empty-test/input | 1 -
.../resources/decoder/empty-test/output.gold | 1 -
.../test/resources/decoder/empty-test/test.sh | 29 -
.../test/resources/decoder/too-long/output.gold | 4 -
.../src/test/resources/decoder/too-long/test.sh | 36 -
scripts/compat/sevenize_my_conf_plz.py | 49 +-
74 files changed, 2154 insertions(+), 9592 deletions(-)
----------------------------------------------------------------------
[20/21] incubator-joshua git commit: Renamed TargetBigram to
TargetBigramTest
Posted by mj...@apache.org.
Renamed TargetBigram to TargetBigramTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/89725be5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/89725be5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/89725be5
Branch: refs/heads/7_confsystem
Commit: 89725be589e5901549d0e1c560b31ca296d41851
Parents: 1c423a2
Author: Hieber, Felix <fh...@amazon.de>
Authored: Mon Sep 19 10:48:16 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 10:48:16 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/TargetBigram.java | 80 --------------------
.../joshua/decoder/cky/TargetBigramTest.java | 80 ++++++++++++++++++++
2 files changed, 80 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/89725be5/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
deleted file mode 100644
index 6bba8c6..0000000
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder.cky;
-
-import static com.typesafe.config.ConfigFactory.parseString;
-import static org.apache.joshua.decoder.cky.TestUtil.translate;
-import static org.testng.Assert.assertEquals;
-
-import org.apache.joshua.decoder.Decoder;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.Test;
-
-import com.typesafe.config.Config;
-
-public class TargetBigram {
-
- private static final String INPUT = "this is a test";
- private static final String GOLD_TOPN2 = "this is a test ||| glue_0=-4.000000 TargetBigram_this_is=1.000000 TargetBigram_is_UNK=1.000000 TargetBigram_UNK_UNK=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_this=1.000000 ||| 0.000";
- private static final String GOLD_TOPN3_THRESHOLD20 = "this is a test ||| glue_0=-4.000000 TargetBigram_a_UNK=1.000000 TargetBigram_UNK_a=1.000000 TargetBigram_UNK_UNK=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_UNK=1.000000 ||| 0.000";
- private static final String GOLD_THRESHOLD10 = "this is a test ||| glue_0=-4.000000 TargetBigram_is_a=1.000000 TargetBigram_a_UNK=1.000000 TargetBigram_UNK_is=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_UNK=1.000000 ||| 0.000";
-
- private static final String VOCAB_PATH = "src/test/resources/decoder/target-bigram/vocab";
- private static final String CONF_TOPN2 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
- + VOCAB_PATH + ", top-n = 2 } ]";
- private static final String CONF_TOPN3_THRESHOLD20 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
- + VOCAB_PATH + ", top-n = 3, threshold = 20 } ]";
- private static final String CONF_THRESHOLD10 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
- + VOCAB_PATH + ", threshold = 10 } ]";
-
- private Decoder decoder;
-
- @Test
- public void givenInput_whenDecodingWithTargetBigramAndTopN2_thenOutputCorrect() {
- setUp(CONF_TOPN2);
- String output = translate(INPUT, decoder).trim();
- assertEquals(output, GOLD_TOPN2);
- }
-
- @Test
- public void givenInput_whenDecodingWithTargetBigramAndTopN3Threshold20_thenOutputCorrect() {
- setUp(CONF_TOPN3_THRESHOLD20);
- String output = translate(INPUT, decoder).trim();
- assertEquals(output, GOLD_TOPN3_THRESHOLD20);
- }
-
- @Test
- public void givenInput_whenDecodingWithTargetBigramThreshold10_thenOutputCorrect2() {
- setUp(CONF_THRESHOLD10);
- String output = translate(INPUT, decoder).trim();
- assertEquals(output, GOLD_THRESHOLD10);
- }
-
- public void setUp(String configuration) {
- Config config = parseString(configuration).withFallback(Decoder.getDefaultFlags());
- decoder = new Decoder(config);
- }
-
- @AfterMethod
- public void tearDown() throws Exception {
- decoder.cleanUp();
- decoder = null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/89725be5/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigramTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigramTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigramTest.java
new file mode 100644
index 0000000..fdfbb11
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigramTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.cky;
+
+import static com.typesafe.config.ConfigFactory.parseString;
+import static org.apache.joshua.decoder.cky.TestUtil.translate;
+import static org.testng.Assert.assertEquals;
+
+import org.apache.joshua.decoder.Decoder;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.Test;
+
+import com.typesafe.config.Config;
+
+public class TargetBigramTest {
+
+ private static final String INPUT = "this is a test";
+ private static final String GOLD_TOPN2 = "this is a test ||| glue_0=-4.000000 TargetBigram_this_is=1.000000 TargetBigram_is_UNK=1.000000 TargetBigram_UNK_UNK=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_this=1.000000 ||| 0.000";
+ private static final String GOLD_TOPN3_THRESHOLD20 = "this is a test ||| glue_0=-4.000000 TargetBigram_a_UNK=1.000000 TargetBigram_UNK_a=1.000000 TargetBigram_UNK_UNK=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_UNK=1.000000 ||| 0.000";
+ private static final String GOLD_THRESHOLD10 = "this is a test ||| glue_0=-4.000000 TargetBigram_is_a=1.000000 TargetBigram_a_UNK=1.000000 TargetBigram_UNK_is=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_UNK=1.000000 ||| 0.000";
+
+ private static final String VOCAB_PATH = "src/test/resources/decoder/target-bigram/vocab";
+ private static final String CONF_TOPN2 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
+ + VOCAB_PATH + ", top-n = 2 } ]";
+ private static final String CONF_TOPN3_THRESHOLD20 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
+ + VOCAB_PATH + ", top-n = 3, threshold = 20 } ]";
+ private static final String CONF_THRESHOLD10 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
+ + VOCAB_PATH + ", threshold = 10 } ]";
+
+ private Decoder decoder;
+
+ @Test
+ public void givenInput_whenDecodingWithTargetBigramAndTopN2_thenOutputCorrect() {
+ setUp(CONF_TOPN2);
+ String output = translate(INPUT, decoder).trim();
+ assertEquals(output, GOLD_TOPN2);
+ }
+
+ @Test
+ public void givenInput_whenDecodingWithTargetBigramAndTopN3Threshold20_thenOutputCorrect() {
+ setUp(CONF_TOPN3_THRESHOLD20);
+ String output = translate(INPUT, decoder).trim();
+ assertEquals(output, GOLD_TOPN3_THRESHOLD20);
+ }
+
+ @Test
+ public void givenInput_whenDecodingWithTargetBigramThreshold10_thenOutputCorrect2() {
+ setUp(CONF_THRESHOLD10);
+ String output = translate(INPUT, decoder).trim();
+ assertEquals(output, GOLD_THRESHOLD10);
+ }
+
+ public void setUp(String configuration) {
+ Config config = parseString(configuration).withFallback(Decoder.getDefaultFlags());
+ decoder = new Decoder(config);
+ }
+
+ @AfterMethod
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ decoder = null;
+ }
+
+}
[12/21] incubator-joshua git commit: Fixed
SourceAnnotationsNotUsingTest by adding missing wordpenalty feature function
in conf
Posted by mj...@apache.org.
Fixed SourceAnnotationsNotUsingTest by adding missing wordpenalty feature function in conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/9af0dea3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/9af0dea3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/9af0dea3
Branch: refs/heads/7_confsystem
Commit: 9af0dea3b0d541731cc044696a2d9841f9e1c2b6
Parents: 5a5c6d9
Author: Hieber, Felix <fh...@amazon.de>
Authored: Mon Sep 19 08:23:02 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 08:23:02 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9af0dea3/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
index a37e9b0..cf27edb 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
@@ -11,6 +11,7 @@ feature_function = WordPenalty
feature_functions = [
{ class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
+ { class = WordPenalty }
]
grammars = [
[13/21] incubator-joshua git commit: Fixed more tests with feature
functions erronously left in old format
Posted by mj...@apache.org.
Fixed more tests with feature functions erronously left in old format
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/0467f3df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/0467f3df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/0467f3df
Branch: refs/heads/7_confsystem
Commit: 0467f3df6e7f4ed6751b6dcff11d710449101a87
Parents: 9af0dea
Author: Hieber, Felix <fh...@amazon.de>
Authored: Mon Sep 19 08:26:01 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 08:26:01 2016 +0200
----------------------------------------------------------------------
.../cky/SourceAnnotationsNotUsingTest.conf | 3 +-
.../decoder/cky/SourceAnnotationsUsingTest.conf | 4 +--
.../decoder/cky/UniqueHypothesesTest.conf | 32 ++++++++++++++++++++
.../decoder/cky/UniqueHypothesesTest.config | 32 --------------------
.../decoder/cky/UniqueHypothesesTest.java | 2 +-
5 files changed, 36 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0467f3df/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
index cf27edb..987e912 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
@@ -6,12 +6,11 @@ top_n = 1
use_unique_nbest = true
output_format = %s ||| %f ||| %c
include_align_index = false
-feature_function = OOVPenalty
-feature_function = WordPenalty
feature_functions = [
{ class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
{ class = WordPenalty }
+ { class = OOVPenalty }
]
grammars = [
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0467f3df/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
index 7533ee0..b5b3a8e 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
@@ -6,12 +6,12 @@ top_n = 1
use_unique_nbest = true
output_format = %s ||| %f --- %c
include_align_index = false
-feature_function = OOVPenalty
-feature_function = WordPenalty
source_annotations = true
feature_functions = [
{ class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
+ { class = WordPenalty }
+ { class = OOVPenalty }
]
grammars = [
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0467f3df/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.conf
new file mode 100644
index 0000000..2300ff8
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.conf
@@ -0,0 +1,32 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = false
+pop_limit = 100
+top_n = 300
+use_unique_nbest = true
+output_format = %s
+include_align_index = false
+
+feature_functions = [
+ { class = StateMinimizingLanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/phrase/unique-hypotheses/lm.1.gz }
+ { class = OOVPenalty }
+ { class = WordPenalty }
+ { class = Distortion }
+ { class = PhrasePenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 0, path = src/test/resources/decoder/phrase/unique-hypotheses/rules.1.gz }
+]
+
+weights = {
+ Distortion = 1.0
+ OOVPenalty = 1.0
+ PhrasePenalty = 1.0
+ WordPenalty = -2.844814
+ lm_0 = 1.0
+ tm_pt_0 = 1.0
+ tm_pt_1 = 1.0
+ tm_pt_2 = 1.0
+ tm_pt_3 = 1.0
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0467f3df/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
deleted file mode 100644
index b7172ef..0000000
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
+++ /dev/null
@@ -1,32 +0,0 @@
-default_non_terminal = X
-goal_symbol = GOAL
-mark_oovs = false
-pop_limit = 100
-top_n = 300
-use_unique_nbest = true
-output_format = %s
-include_align_index = false
-feature_function = Distortion
-feature_function = PhrasePenalty
-
-feature_functions = [
- { class = StateMinimizingLanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/phrase/unique-hypotheses/lm.1.gz }
- { class = OOVPenalty }
- { class = WordPenalty }
-]
-
-grammars = [
- { class = TextGrammar, owner = pt, span_limit = 0, path = src/test/resources/decoder/phrase/unique-hypotheses/rules.1.gz }
-]
-
-weights = {
- Distortion = 1.0
- OOVPenalty = 1.0
- PhrasePenalty = 1.0
- WordPenalty = -2.844814
- lm_0 = 1.0
- tm_pt_0 = 1.0
- tm_pt_1 = 1.0
- tm_pt_2 = 1.0
- tm_pt_3 = 1.0
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0467f3df/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
index 6e163e5..5ad4b64 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
@@ -46,7 +46,7 @@ public class UniqueHypothesesTest {
@Test
public void givenInputSentence_whenDecodingWithUniqueHypotheses_thenAllHypothesesUnique()
throws Exception {
- Config config = parseResources(this.getClass(), "UniqueHypothesesTest.config")
+ Config config = parseResources(this.getClass(), "UniqueHypothesesTest.conf")
.withFallback(Decoder.getDefaultFlags());
KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
[18/21] incubator-joshua git commit: Merge pull request #4 from
michael-aloys/7_confsystem
Posted by mj...@apache.org.
Merge pull request #4 from michael-aloys/7_confsystem
Fixed test to work with new configuration (2)
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/6de867e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/6de867e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/6de867e9
Branch: refs/heads/7_confsystem
Commit: 6de867e904dcc6c9e7b180251ff5f4347b211a74
Parents: 547e0b9 39f8485
Author: Felix Hieber <fh...@users.noreply.github.com>
Authored: Mon Sep 19 10:37:33 2016 +0200
Committer: GitHub <no...@github.com>
Committed: Mon Sep 19 10:37:33 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/TargetBigram.java | 37 +++++++++++---------
scripts/compat/sevenize_my_conf_plz.py | 2 +-
2 files changed, 22 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[04/21] incubator-joshua git commit: Added TestUtil method that
covers redundant test code for former regressions tests,
i.e. load input strings, decode, load gold, assert equals decoded and gold
Posted by mj...@apache.org.
Added TestUtil method that covers redundant test code for former regressions tests, i.e. load input strings, decode, load gold, assert equals decoded and gold
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/89a112a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/89a112a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/89a112a0
Branch: refs/heads/7_confsystem
Commit: 89a112a07c851cdc9eefb93e5f5d915363031356
Parents: 28f4810
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 18:40:57 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 18:40:57 2016 +0200
----------------------------------------------------------------------
.../org/apache/joshua/decoder/cky/TestUtil.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/89a112a0/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
index 0855e99..8781092 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
@@ -14,6 +14,11 @@
*/
package org.apache.joshua.decoder.cky;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
+import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
+import static org.testng.Assert.assertEquals;
+
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
@@ -35,7 +40,7 @@ public class TestUtil {
/**
* Loads a text file and returns a list containing one string per line in the file.
*
- * @param pathToFile
+ * @param pathToFile path to file on disc
* @return
* @throws IOException
*/
@@ -77,4 +82,13 @@ public class TestUtil {
final Sentence sentence = new Sentence(input, 0, decoder.getFlags());
return decoder.decode(sentence).toString();
}
+
+ public static void decodeAndAssertDecodedOutputEqualsGold(String pathToInput, Decoder decoder, String pathToGold) throws IOException {
+ List<String> inputStrings = loadStringsFromFile(pathToInput);
+
+ List<String> decodedStrings = decodeList(inputStrings, decoder);
+
+ List<String> goldStrings = loadStringsFromFile(pathToGold);
+ assertEquals(decodedStrings, goldStrings);
+ }
}
[08/21] incubator-joshua git commit: Fixed UniqueHypothesesTest,
SourceAnnotationsTest and OOVListTest to work with new configuration.
Compiles but does not pass yet.
Posted by mj...@apache.org.
Fixed UniqueHypothesesTest, SourceAnnotationsTest and OOVListTest to work with new configuration. Compiles but does not pass yet.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/3ed109af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/3ed109af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/3ed109af
Branch: refs/heads/7_confsystem
Commit: 3ed109afab7c91dc93a92a387053bc140029784b
Parents: 3b70a97
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 19:20:12 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 19:23:35 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/OOVListTest.conf | 28 ++++++++++++++++
.../apache/joshua/decoder/cky/OOVListTest.gold | 3 ++
.../apache/joshua/decoder/cky/OOVListTest.in | 3 ++
.../apache/joshua/decoder/cky/OOVListTest.java | 34 ++++++--------------
.../cky/SourceAnnotationsNotUsingTest.conf | 27 ++++++++++++++++
.../decoder/cky/SourceAnnotationsTest.java | 28 ++++++++--------
.../decoder/cky/SourceAnnotationsUsingTest.conf | 28 ++++++++++++++++
.../decoder/cky/UniqueHypothesesTest.config | 32 ++++++++++++++++++
.../decoder/cky/UniqueHypothesesTest.java | 19 +++++------
9 files changed, 152 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf
new file mode 100644
index 0000000..d045e22
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.conf
@@ -0,0 +1,28 @@
+mark_oovs = true
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+use_unique_nbest = true
+use_tree_nbest = false
+top_n = 1
+oov_list = CD 0.0488752 JJ 0.186114 NN 0.291795 NNS 0.0894967 NP 0.117171 OOV 0.033015 VB 0.0313967 VBG 0.0404596 VBN 0.0317203
+output_format = %s ||| %f ||| %c
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/oov-list/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = phrase, span_limit = 20, path = src/test/resources/decoder/oov-list/grammar }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/oov-list/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = -3.6942747832593694
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ phrase_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold
new file mode 100644
index 0000000..7cebb7b
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.gold
@@ -0,0 +1,3 @@
+Goats eat cheese ||| phrase_0=0.000000 lm_0=-16.587435 glue_0=-1.000000 WordPenalty=-2.171473 ||| -11.503
+i will go home ||| phrase_0=0.000000 lm_0=-12.155182 glue_0=-1.000000 WordPenalty=-2.605767 ||| -4.414
+goets_OOV eet_OOV cheez_OOV ||| phrase_0=0.000000 lm_0=-17.699974 glue_0=-1.000000 WordPenalty=-2.171473 OOVPenalty=-7.749294 ||| -20.629
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in
new file mode 100644
index 0000000..23fd45a
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.in
@@ -0,0 +1,3 @@
+CHEESE GOATS EAT
+3 1 4 2
+goets eet cheez
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
index 86282bf..394d806 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
@@ -18,20 +18,17 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class OOVListTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -44,23 +41,12 @@ public class OOVListTest {
@Test
public void givenInput_whenDecodingWithOOVList_thenScoreAndTranslationCorrect() throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/oov-list/input.txt");
-
- // When
- configureDecoder("src/test/resources/decoder/oov-list/joshua.config");
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/oov-list/output.gold");
- assertEquals(decodedStrings, goldStrings);
- }
+ String inputPath = this.getClass().getResource("OOVListTest.in").getFile();
+ String goldPath = this.getClass().getResource("OOVListTest.gold").getFile();
+ Config config = parseResources(this.getClass(), "OOVListTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- public void configureDecoder(String pathToConfig) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
new file mode 100644
index 0000000..a37e9b0
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsNotUsingTest.conf
@@ -0,0 +1,27 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = true
+pop_limit = 100
+top_n = 1
+use_unique_nbest = true
+output_format = %s ||| %f ||| %c
+include_align_index = false
+feature_function = OOVPenalty
+feature_function = WordPenalty
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 20, path = src/test/resources/decoder/source-annotations/grammar }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/source-annotations/grammar.glue }
+]
+
+weights = {
+ OOVPenalty = 1
+ WordPenalty = -1.5244636836685694
+ glue_0 = -0.1663815584150378
+ lm_0 = 1.0
+ pt_0 = -0.049141264495762726
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
index 28fc9e3..ee0f7aa 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsTest.java
@@ -18,45 +18,43 @@
*/
package org.apache.joshua.decoder.cky;
+import static com.typesafe.config.ConfigFactory.parseResources;
import static org.apache.joshua.decoder.cky.TestUtil.translate;
import static org.testng.Assert.assertEquals;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class SourceAnnotationsTest {
private static final String INPUT = "mis[tag=ADJ;num=PL;class=OOV] amigos me llaman";
private static final String GOLD_WITHOUT_ANNOTATIONS = "my friends call me ||| pt_0=3.000000 lm_0=-11.973694 glue_0=-3.000000 WordPenalty=-2.605767 ||| -7.650";
private static final String GOLD_WITH_ANNOTATIONS = "my friends call me ||| pt_0=3.000000 lm_0=-111.512733 glue_0=-3.000000 WordPenalty=-2.605767 ||| -107.189";
- private static final String JOSHUA_CONFIG_PATH = "src/test/resources/decoder/source-annotations/joshua.config";
-
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@Test
public void givenInput_whenNotUsingSourceAnnotations_thenOutputCorrect() throws Exception {
- setUp(false);
- String output = translate(INPUT, decoder, joshuaConfig);
+ Config config = parseResources(this.getClass(), "SourceAnnotationsNotUsingTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
+
+ String output = translate(INPUT, decoder);
assertEquals(output.trim(), GOLD_WITHOUT_ANNOTATIONS);
}
@Test
public void givenInput_whenUsingSourceAnnotations_thenOutputCorrect() throws Exception {
- setUp(true);
- String output = translate(INPUT, decoder, joshuaConfig);
- assertEquals(output.trim(), GOLD_WITH_ANNOTATIONS);
- }
+ Config config = parseResources(this.getClass(), "SourceAnnotationsUsingTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- public void setUp(boolean sourceAnnotations) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(JOSHUA_CONFIG_PATH);
- joshuaConfig.source_annotations = sourceAnnotations;
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
+ String output = translate(INPUT, decoder);
+ assertEquals(output.trim(), GOLD_WITH_ANNOTATIONS);
}
@AfterMethod
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
new file mode 100644
index 0000000..7533ee0
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
@@ -0,0 +1,28 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = true
+pop_limit = 100
+top_n = 1
+use_unique_nbest = true
+output_format = %s ||| %f --- %c
+include_align_index = false
+feature_function = OOVPenalty
+feature_function = WordPenalty
+source_annotations = true
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 20, path = src/test/resources/decoder/source-annotations/grammar }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/source-annotations/grammar.glue }
+]
+
+weights = {
+ OOVPenalty = 1
+ WordPenalty = -1.5244636836685694
+ glue_0 = -0.1663815584150378
+ lm_0 = 1.0
+ pt_0 = -0.049141264495762726
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
new file mode 100644
index 0000000..b7172ef
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.config
@@ -0,0 +1,32 @@
+default_non_terminal = X
+goal_symbol = GOAL
+mark_oovs = false
+pop_limit = 100
+top_n = 300
+use_unique_nbest = true
+output_format = %s
+include_align_index = false
+feature_function = Distortion
+feature_function = PhrasePenalty
+
+feature_functions = [
+ { class = StateMinimizingLanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/phrase/unique-hypotheses/lm.1.gz }
+ { class = OOVPenalty }
+ { class = WordPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 0, path = src/test/resources/decoder/phrase/unique-hypotheses/rules.1.gz }
+]
+
+weights = {
+ Distortion = 1.0
+ OOVPenalty = 1.0
+ PhrasePenalty = 1.0
+ WordPenalty = -2.844814
+ lm_0 = 1.0
+ tm_pt_0 = 1.0
+ tm_pt_1 = 1.0
+ tm_pt_2 = 1.0
+ tm_pt_3 = 1.0
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3ed109af/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
index 825de45..6e163e5 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/UniqueHypothesesTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.joshua.decoder.cky;
+import static com.typesafe.config.ConfigFactory.parseResources;
import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
import static org.testng.Assert.assertEquals;
@@ -27,11 +28,12 @@ import java.util.List;
import java.util.Set;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
/**
* Ensures that derivations are unique for the phrase-based decoder.
*/
@@ -39,15 +41,16 @@ public class UniqueHypothesesTest {
public static final String INPUT = "una estrategia republicana para obstaculizar la reelecci�n de Obama";
- private JoshuaConfiguration joshuaConfig = null;
private Decoder decoder = null;
@Test
public void givenInputSentence_whenDecodingWithUniqueHypotheses_thenAllHypothesesUnique()
throws Exception {
- configureDecoder("src/test/resources/decoder/phrase/unique-hypotheses/joshua.config");
- List<String> decodedStrings = decodeList(Arrays.asList(new String[] { INPUT }), decoder,
- joshuaConfig);
+ Config config = parseResources(this.getClass(), "UniqueHypothesesTest.config")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
+
+ List<String> decodedStrings = decodeList(Arrays.asList(new String[] { INPUT }), decoder);
assertEquals(decodedStrings.size(), 300);
@@ -57,12 +60,6 @@ public class UniqueHypothesesTest {
assertEquals(decodedStrings.size(), uniqueDecodedStrings.size());
}
- public void configureDecoder(String pathToConfig) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
-
@AfterMethod
public void tearDown() throws Exception {
if (decoder != null) {
[07/21] incubator-joshua git commit: Fixed TreeOutputTest to work
with new configuration.
Posted by mj...@apache.org.
Fixed TreeOutputTest to work with new configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/3b70a978
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/3b70a978
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/3b70a978
Branch: refs/heads/7_confsystem
Commit: 3b70a978462fc98d3b6336c6498f339391900879
Parents: a7e2adf
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 19:14:12 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 19:14:12 2016 +0200
----------------------------------------------------------------------
.../joshua/decoder/cky/TreeOutputTest.conf | 42 ++++++++++++++++++++
.../joshua/decoder/cky/TreeOutputTest.gold | 5 +++
.../apache/joshua/decoder/cky/TreeOutputTest.in | 5 +++
.../joshua/decoder/cky/TreeOutputTest.java | 32 +++++----------
4 files changed, 62 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3b70a978/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.conf
new file mode 100644
index 0000000..e5cd475
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.conf
@@ -0,0 +1,42 @@
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %t
+use_unique_nbest = true
+top_n = 1
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/tree-output/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/tree-output/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/tree-output/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = -3.6942747832593694
+ lm_0 = 1.2373676802179452
+ tm_glue_0 = 1
+ tm_pt_0 = -2.4497429277910214
+ tm_pt_1 = 0.7224581556224123
+ tm_pt_10 = 0.2593091306160006
+ tm_pt_11 = 0.25597137004462134
+ tm_pt_12 = 0.3538894647790496
+ tm_pt_13 = -0.36212061186692646
+ tm_pt_14 = -0.32923261148678096
+ tm_pt_15 = 0.5524863522177359
+ tm_pt_16 = 0.23451595442127693
+ tm_pt_2 = -0.31689069155153504
+ tm_pt_3 = 0.33861043967238036
+ tm_pt_4 = 0.03553113401320236
+ tm_pt_5 = 0.19138972284064748
+ tm_pt_6 = 0.3417994095521415
+ tm_pt_7 = -0.9936312455671283
+ tm_pt_8 = 0.9070737587091975
+ tm_pt_9 = 0.8202511858619419
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3b70a978/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.gold
new file mode 100644
index 0000000..abbeb79
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.gold
@@ -0,0 +1,5 @@
+(GOAL{0-5} (GOAL{0-4} (GOAL{0-3} (GOAL{0-1} <s>) (A{1-3} (B{1-2} foo) (C{2-3} bar))) (D{3-4} baz)) </s>)
+()
+(GOAL{0-3} (GOAL{0-2} (GOAL{0-1} <s>) (D{1-2} baz)) </s>)
+(GOAL{0-4} (GOAL{0-3} (GOAL{0-1} <s>) (S{1-3} I AM)) </s>)
+(GOAL{0-4} <s> (NP\DT{1-3} right (NN{2-3} xslot)) </s>)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3b70a978/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.in
new file mode 100644
index 0000000..a5a18b4
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.in
@@ -0,0 +1,5 @@
+foo bar baz
+an unparseable sentence
+baz
+yo soy
+purchase xslot
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/3b70a978/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.java
index 578790d..0efb3c1 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TreeOutputTest.java
@@ -18,20 +18,17 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class TreeOutputTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -44,22 +41,13 @@ public class TreeOutputTest {
@Test
public void givenInput_whenDecodingWithTreeOutput_thenOutputCorrect() throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile("src/test/resources/decoder/tree-output/input");
+ String inputPath = this.getClass().getResource("TreeOutputTest.in").getFile();
+ String goldPath = this.getClass().getResource("TreeOutputTest.gold").getFile();
+ Config config = parseResources(this.getClass(), "TreeOutputTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- // When
- configureDecoder("src/test/resources/decoder/tree-output/joshua.config");
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/tree-output/output.gold");
- assertEquals(decodedStrings, goldStrings);
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
- public void configureDecoder(String pathToConfig) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
}
[06/21] incubator-joshua git commit: Fixed RescoringTest to work with
new configuration.
Posted by mj...@apache.org.
Fixed RescoringTest to work with new configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/a7e2adf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/a7e2adf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/a7e2adf4
Branch: refs/heads/7_confsystem
Commit: a7e2adf4faafbdf4b8ba75b8c67ecf62c62ab56a
Parents: d117c17
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 18:58:02 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 18:58:02 2016 +0200
----------------------------------------------------------------------
.../joshua/decoder/cky/RescoringTest.conf | 28 ++++++++++++++++
.../joshua/decoder/cky/RescoringTest.gold | 12 +++++++
.../apache/joshua/decoder/cky/RescoringTest.in | 2 ++
.../joshua/decoder/cky/RescoringTest.java | 35 ++++++--------------
4 files changed, 52 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf
new file mode 100644
index 0000000..8c8a751
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.conf
@@ -0,0 +1,28 @@
+rescore_forest = true
+rescore_forest_weight = 100
+mark_oovs = true
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %s ||| %f ||| %c
+use_unique_nbest = true
+top_n = 2
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/rescoring/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/rescoring/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/rescoring/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = -1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold
new file mode 100644
index 0000000..6ed6cd2
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.gold
@@ -0,0 +1,12 @@
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the kid eated the cockroach ||| pt_0=15.000000 lm_0=-20.053476 glue_0=-5.000000 WordPenalty=-3.040061 ||| -31.773
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
+the boy ate the cockroach ||| pt_0=6.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -19.240
+the boy eated the cockroach ||| pt_0=11.000000 lm_0=-17.198177 glue_0=-5.000000 WordPenalty=-3.040061 ||| -24.240
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in
new file mode 100644
index 0000000..5562a01
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.in
@@ -0,0 +1,2 @@
+el nino tomo la cucaracha ||| ||| the boy ate the cockroach
+el nino tomo la cucaracha ||| ||| the big storm swarmed the coast ||| the big storm only swarmed the coast
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a7e2adf4/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
index 7f4cdc0..de59d08 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/RescoringTest.java
@@ -18,22 +18,17 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class RescoringTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -47,23 +42,13 @@ public class RescoringTest {
@Test
public void givenInput_whenDecodingWithRescoring_thenScoreAndTranslationCorrect()
throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/rescoring/input.txt");
+ String inputPath = this.getClass().getResource("RescoringTest.in").getFile();
+ String goldPath = this.getClass().getResource("RescoringTest.gold").getFile();
+ Config config = parseResources(this.getClass(), "RescoringTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- // When
- configureDecoder("src/test/resources/decoder/rescoring/joshua.config");
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/rescoring/output.gold");
- assertEquals(decodedStrings, goldStrings);
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
- public void configureDecoder(String pathToConfig) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
}
[14/21] incubator-joshua git commit: Fixed SourceAnnotationsUsingTest
config
Posted by mj...@apache.org.
Fixed SourceAnnotationsUsingTest config
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/6e0bb3b2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/6e0bb3b2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/6e0bb3b2
Branch: refs/heads/7_confsystem
Commit: 6e0bb3b20952eb48db36eda443f14efa71bbcc3d
Parents: 0467f3d
Author: Hieber, Felix <fh...@amazon.de>
Authored: Mon Sep 19 08:33:24 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 08:33:24 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6e0bb3b2/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
index b5b3a8e..8b4c20d 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/SourceAnnotationsUsingTest.conf
@@ -4,12 +4,11 @@ mark_oovs = true
pop_limit = 100
top_n = 1
use_unique_nbest = true
-output_format = %s ||| %f --- %c
+output_format = %s ||| %f ||| %c
include_align_index = false
-source_annotations = true
feature_functions = [
- { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, source_annotations = true, lm_file = src/test/resources/decoder/source-annotations/lm.kenlm }
{ class = WordPenalty }
{ class = OOVPenalty }
]
[09/21] incubator-joshua git commit: Removed unnecessary imports.
Posted by mj...@apache.org.
Removed unnecessary imports.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/0ede005e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/0ede005e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/0ede005e
Branch: refs/heads/7_confsystem
Commit: 0ede005e4c8c4601128b292cf3e02f7b3d2b2262
Parents: 3ed109a
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 19:34:08 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 19:34:08 2016 +0200
----------------------------------------------------------------------
.../org/apache/joshua/decoder/cky/TestUtil.java | 47 ++++++++++----------
1 file changed, 24 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/0ede005e/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
index 8781092..c0a8c88 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
@@ -14,9 +14,6 @@
*/
package org.apache.joshua.decoder.cky;
-import static com.typesafe.config.ConfigFactory.parseResources;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
import static org.testng.Assert.assertEquals;
import java.io.IOException;
@@ -29,18 +26,17 @@ import java.util.stream.Collectors;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.segment_file.Sentence;
-import org.apache.joshua.util.io.KenLmTestUtil;
-
-import com.typesafe.config.Config;
public class TestUtil {
public static final String N_BEST_SEPARATOR = "\n";
-
+
/**
- * Loads a text file and returns a list containing one string per line in the file.
+ * Loads a text file and returns a list containing one string per line in the
+ * file.
*
- * @param pathToFile path to file on disc
+ * @param pathToFile
+ * path to file on disc
* @return
* @throws IOException
*/
@@ -48,15 +44,18 @@ public class TestUtil {
List<String> inputLines = Files.lines(Paths.get(pathToFile)).collect(Collectors.toList());
return inputLines;
}
-
+
/**
*
- * @param inputStrings A list of strings that should be decoded,
- * @param decoder An initialized decoder,
- * @param joshuaConfig The JoshuaConfiguration corresponding to the decoder.
- * @return A list of decoded strings. If the decoder produces a n-best list (separated by
- * N_BEST_SEPARATOR), then each translation of the n-best list has its own entry in the
- * returned list.
+ * @param inputStrings
+ * A list of strings that should be decoded,
+ * @param decoder
+ * An initialized decoder,
+ * @param joshuaConfig
+ * The JoshuaConfiguration corresponding to the decoder.
+ * @return A list of decoded strings. If the decoder produces a n-best list
+ * (separated by N_BEST_SEPARATOR), then each translation of the
+ * n-best list has its own entry in the returned list.
*/
public static List<String> decodeList(List<String> inputStrings, Decoder decoder) {
final List<String> decodedStrings = new ArrayList<>();
@@ -71,7 +70,8 @@ public class TestUtil {
}
/**
- * Translates the given input string and returns the translation converted into a string.
+ * Translates the given input string and returns the translation converted
+ * into a string.
*
* @param input
* @param decoder
@@ -82,13 +82,14 @@ public class TestUtil {
final Sentence sentence = new Sentence(input, 0, decoder.getFlags());
return decoder.decode(sentence).toString();
}
-
- public static void decodeAndAssertDecodedOutputEqualsGold(String pathToInput, Decoder decoder, String pathToGold) throws IOException {
- List<String> inputStrings = loadStringsFromFile(pathToInput);
- List<String> decodedStrings = decodeList(inputStrings, decoder);
+ public static void decodeAndAssertDecodedOutputEqualsGold(String pathToInput, Decoder decoder,
+ String pathToGold) throws IOException {
+ List<String> inputStrings = loadStringsFromFile(pathToInput);
+
+ List<String> decodedStrings = decodeList(inputStrings, decoder);
- List<String> goldStrings = loadStringsFromFile(pathToGold);
- assertEquals(decodedStrings, goldStrings);
+ List<String> goldStrings = loadStringsFromFile(pathToGold);
+ assertEquals(decodedStrings, goldStrings);
}
}
[19/21] incubator-joshua git commit: Fixed TargetBigram test by
changing expected gold outputs (flipped glue score and sorted feature output)
Posted by mj...@apache.org.
Fixed TargetBigram test by changing expected gold outputs (flipped glue score and sorted feature output)
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/1c423a2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/1c423a2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/1c423a2b
Branch: refs/heads/7_confsystem
Commit: 1c423a2b76ba1ecbdf45b5d2eac20d2270f0b853
Parents: 6de867e
Author: Hieber, Felix <fh...@amazon.de>
Authored: Mon Sep 19 10:47:48 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 10:47:48 2016 +0200
----------------------------------------------------------------------
.../apache/joshua/decoder/cky/TargetBigram.java | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1c423a2b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
index add96ef..6bba8c6 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TargetBigram.java
@@ -31,9 +31,9 @@ import com.typesafe.config.Config;
public class TargetBigram {
private static final String INPUT = "this is a test";
- private static final String GOLD_TOPN2 = "this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_this=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_UNK=1.000 TargetBigram_is_UNK=1.000 TargetBigram_this_is=1.000 ||| 0.000";
- private static final String GOLD_TOPN3_THRESHOLD20 = "this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_UNK=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_UNK=1.000 TargetBigram_UNK_a=1.000 TargetBigram_a_UNK=1.000 ||| 0.000";
- private static final String GOLD_THRESHOLD10 = "this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_UNK=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_is=1.000 TargetBigram_a_UNK=1.000 TargetBigram_is_a=1.000 ||| 0.000";
+ private static final String GOLD_TOPN2 = "this is a test ||| glue_0=-4.000000 TargetBigram_this_is=1.000000 TargetBigram_is_UNK=1.000000 TargetBigram_UNK_UNK=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_this=1.000000 ||| 0.000";
+ private static final String GOLD_TOPN3_THRESHOLD20 = "this is a test ||| glue_0=-4.000000 TargetBigram_a_UNK=1.000000 TargetBigram_UNK_a=1.000000 TargetBigram_UNK_UNK=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_UNK=1.000000 ||| 0.000";
+ private static final String GOLD_THRESHOLD10 = "this is a test ||| glue_0=-4.000000 TargetBigram_is_a=1.000000 TargetBigram_a_UNK=1.000000 TargetBigram_UNK_is=1.000000 TargetBigram_UNK_</s>=1.000000 TargetBigram_<s>_UNK=1.000000 ||| 0.000";
private static final String VOCAB_PATH = "src/test/resources/decoder/target-bigram/vocab";
private static final String CONF_TOPN2 = "output_format = %s ||| %f ||| %c \n feature_functions = [ { class = TargetBigram, vocab = "
@@ -48,22 +48,22 @@ public class TargetBigram {
@Test
public void givenInput_whenDecodingWithTargetBigramAndTopN2_thenOutputCorrect() {
setUp(CONF_TOPN2);
- String output = translate(INPUT, decoder);
- assertEquals(output.trim(), GOLD_TOPN2);
+ String output = translate(INPUT, decoder).trim();
+ assertEquals(output, GOLD_TOPN2);
}
@Test
public void givenInput_whenDecodingWithTargetBigramAndTopN3Threshold20_thenOutputCorrect() {
setUp(CONF_TOPN3_THRESHOLD20);
- String output = translate(INPUT, decoder);
- assertEquals(output.trim(), GOLD_TOPN3_THRESHOLD20);
+ String output = translate(INPUT, decoder).trim();
+ assertEquals(output, GOLD_TOPN3_THRESHOLD20);
}
@Test
public void givenInput_whenDecodingWithTargetBigramThreshold10_thenOutputCorrect2() {
setUp(CONF_THRESHOLD10);
- String output = translate(INPUT, decoder);
- assertEquals(output.trim(), GOLD_THRESHOLD10);
+ String output = translate(INPUT, decoder).trim();
+ assertEquals(output, GOLD_THRESHOLD10);
}
public void setUp(String configuration) {
[10/21] incubator-joshua git commit: Merge pull request #3 from
michael-aloys/7_confsystem
Posted by mj...@apache.org.
Merge pull request #3 from michael-aloys/7_confsystem
Fixed some tests to work with new configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/c425dcf5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/c425dcf5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/c425dcf5
Branch: refs/heads/7_confsystem
Commit: c425dcf556ba59859852afc9892144f8150b4dcd
Parents: c23af30 0ede005
Author: Felix Hieber <fh...@users.noreply.github.com>
Authored: Mon Sep 19 08:06:47 2016 +0200
Committer: GitHub <no...@github.com>
Committed: Mon Sep 19 08:06:47 2016 +0200
----------------------------------------------------------------------
.../org/apache/joshua/decoder/cky/NAryTest.conf | 24 ++++++
.../org/apache/joshua/decoder/cky/NAryTest.gold | 2 +
.../org/apache/joshua/decoder/cky/NAryTest.in | 2 +
.../org/apache/joshua/decoder/cky/NAryTest.java | 49 +++---------
.../NumTranslationOptionsNoDotChartTest.conf | 28 +++++++
.../NumTranslationOptionsNoDotChartTest.gold | 4 +
.../cky/NumTranslationOptionsPackedTest.conf | 27 +++++++
.../cky/NumTranslationOptionsPackedTest.gold | 4 +
.../decoder/cky/NumTranslationOptionsTest.conf | 27 +++++++
.../decoder/cky/NumTranslationOptionsTest.gold | 4 +
.../decoder/cky/NumTranslationOptionsTest.in | 1 +
.../decoder/cky/NumTranslationOptionsTest.java | 83 ++++++--------------
.../apache/joshua/decoder/cky/OOVListTest.conf | 28 +++++++
.../apache/joshua/decoder/cky/OOVListTest.gold | 3 +
.../apache/joshua/decoder/cky/OOVListTest.in | 3 +
.../apache/joshua/decoder/cky/OOVListTest.java | 34 +++-----
.../joshua/decoder/cky/RescoringTest.conf | 28 +++++++
.../joshua/decoder/cky/RescoringTest.gold | 12 +++
.../apache/joshua/decoder/cky/RescoringTest.in | 2 +
.../joshua/decoder/cky/RescoringTest.java | 35 +++------
.../cky/SourceAnnotationsNotUsingTest.conf | 27 +++++++
.../decoder/cky/SourceAnnotationsTest.java | 28 +++----
.../decoder/cky/SourceAnnotationsUsingTest.conf | 28 +++++++
.../org/apache/joshua/decoder/cky/TestUtil.java | 41 +++++++---
.../joshua/decoder/cky/TreeOutputTest.conf | 42 ++++++++++
.../joshua/decoder/cky/TreeOutputTest.gold | 5 ++
.../apache/joshua/decoder/cky/TreeOutputTest.in | 5 ++
.../joshua/decoder/cky/TreeOutputTest.java | 32 +++-----
.../decoder/cky/UniqueHypothesesTest.config | 32 ++++++++
.../decoder/cky/UniqueHypothesesTest.java | 19 ++---
30 files changed, 450 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
[21/21] incubator-joshua git commit: Added missing constrained test
files
Posted by mj...@apache.org.
Added missing constrained test files
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/1d4309ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/1d4309ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/1d4309ae
Branch: refs/heads/7_confsystem
Commit: 1d4309ae1c10454e822cbea5658303770c722a18
Parents: 89725be
Author: Matt Post <po...@cs.jhu.edu>
Authored: Mon Sep 19 06:15:16 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Mon Sep 19 06:15:16 2016 -0400
----------------------------------------------------------------------
.../joshua/decoder/cky/ConstrainedTest.conf | 42 ++++++++++++++++++++
.../joshua/decoder/cky/ConstrainedTest.gold | 30 ++++++++++++++
.../joshua/decoder/cky/ConstrainedTest.in | 8 ++++
3 files changed, 80 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1d4309ae/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.conf
new file mode 100644
index 0000000..0291dbf
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.conf
@@ -0,0 +1,42 @@
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c %s
+use_unique_nbest = true
+top_n = 10
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/constrained/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/constrained/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/constrained/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = -3.6942747832593694
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = 2.4497429277910214
+ pt_1 = -0.7224581556224123
+ pt_10 = -0.2593091306160006
+ pt_11 = -0.25597137004462134
+ pt_12 = -0.3538894647790496
+ pt_13 = 0.36212061186692646
+ pt_14 = 0.32923261148678096
+ pt_15 = -0.5524863522177359
+ pt_16 = -0.23451595442127693
+ pt_2 = 0.31689069155153504
+ pt_3 = -0.33861043967238036
+ pt_4 = -0.03553113401320236
+ pt_5 = -0.19138972284064748
+ pt_6 = -0.3417994095521415
+ pt_7 = 0.9936312455671283
+ pt_8 = -0.9070737587091975
+ pt_9 = -0.8202511858619419
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1d4309ae/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.gold
new file mode 100644
index 0000000..a48d5cb
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.gold
@@ -0,0 +1,30 @@
+-10.471 mathematics so science language .
+-10.543 mathematics is science language .
+-11.587 mathematics that science language .
+-12.065 science mathematics that language .
+-12.259 mathematics so that the science language .
+-12.352 mathematics hence science language .
+-12.590 mathematics that is science language .
+-12.619 mathematics , science language .
+-12.751 mathematics that is why science language .
+-12.755 mathematics in science language .
+-10.543 mathematics is science language .
+-226.302 rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.523 rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.884 rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.201 rabindranath 's birth in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.217 rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.707 rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.982 rabindranath was born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-228.069 rabindranath was born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-228.133 rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-228.160 rabindranath was born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.302 rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.523 rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-230.920 rabindranath born in in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-227.707 rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-232.123 rabindranath born in in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-226.884 rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-231.320 rabindranath was born kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-232.836 rabindranath was born in in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+-229.108 rabindranath born in kolkata was one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1d4309ae/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.in
new file mode 100644
index 0000000..291bca0
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/ConstrainedTest.in
@@ -0,0 +1,8 @@
+\u0997\u09a3\u09bf\u09a4 \u09a4\u09be\u0987 \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09ad\u09be\u09b7\u09be \u0964
+\u0997\u09a3\u09bf\u09a4 \u09a4\u09be\u0987 \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09ad\u09be\u09b7\u09be \u0964 ||| mathematics is science language .
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964 ||| rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964 ||| rabindranath born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964 ||| rabindranath born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964 ||| rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964 ||| rabindranath born in kolkata was one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family
[15/21] incubator-joshua git commit: Disabled OOVListTest since this
functionaliy no longer exists in 7
Posted by mj...@apache.org.
Disabled OOVListTest since this functionaliy no longer exists in 7
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/547e0b98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/547e0b98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/547e0b98
Branch: refs/heads/7_confsystem
Commit: 547e0b98c88c0b17e60147f16856c06a2c7b1a68
Parents: 6e0bb3b
Author: Hieber, Felix <fh...@amazon.de>
Authored: Mon Sep 19 08:37:36 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 08:37:36 2016 +0200
----------------------------------------------------------------------
.../src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/547e0b98/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
index 394d806..5af2812 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/OOVListTest.java
@@ -39,7 +39,8 @@ public class OOVListTest {
}
}
- @Test
+ // TODO(fhieber): Disabled this test as the oov_list functionality was removed in 7 with the new conf system.
+ @Test(enabled = false)
public void givenInput_whenDecodingWithOOVList_thenScoreAndTranslationCorrect() throws Exception {
String inputPath = this.getClass().getResource("OOVListTest.in").getFile();
String goldPath = this.getClass().getResource("OOVListTest.gold").getFile();
[11/21] incubator-joshua git commit: Fixed the
NumTranslationOptionsPacked test by using the default variable from
Decoder.conf
Posted by mj...@apache.org.
Fixed the NumTranslationOptionsPacked test by using the default variable from Decoder.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/5a5c6d98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/5a5c6d98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/5a5c6d98
Branch: refs/heads/7_confsystem
Commit: 5a5c6d987a8916c661803365a9acbe999e910f48
Parents: c425dcf
Author: Hieber, Felix <fh...@amazon.de>
Authored: Sun Sep 18 16:41:57 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Mon Sep 19 08:19:40 2016 +0200
----------------------------------------------------------------------
.../cky/NumTranslationOptionsPackedTest.conf | 2 +-
.../decoder/cky/NumTranslationOptionsTest.java | 3 ++-
.../kbest_extraction/KBestExtractionTest.java | 19 +++++++++----------
3 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5a5c6d98/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
index dc4d6a3..47a7715 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
@@ -14,7 +14,7 @@ feature_functions = [
]
grammars = [
- { class = PackedGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.packed }
+ { class = PackedGrammar, owner = pt, span_limit = 12, rule_cache_size=${default_packed_grammar_cache_size}, path = src/test/resources/decoder/num_translation_options/grammar.packed }
{ class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
]
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5a5c6d98/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
index 66db0ad..ce730be 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
@@ -60,7 +60,8 @@ public class NumTranslationOptionsTest {
String inputPath = this.getClass().getResource(inFile).getFile();
String goldPath = this.getClass().getResource(goldFile).getFile();
Config config = parseResources(this.getClass(), confFile)
- .withFallback(Decoder.getDefaultFlags());
+ .withFallback(Decoder.getDefaultFlags())
+ .resolve();
KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/5a5c6d98/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
index 4a76922..692b74e 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
@@ -18,6 +18,15 @@
*/
package org.apache.joshua.decoder.kbest_extraction;
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
+import static org.testng.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.segment_file.Sentence;
@@ -25,20 +34,10 @@ import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-import org.testng.reporters.Files;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValueFactory;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import static com.google.common.base.Charsets.UTF_8;
-import static java.nio.file.Files.readAllBytes;
-import static org.testng.Assert.assertEquals;
-
/**
* Reimplements the kbest extraction regression test.
*/
[17/21] incubator-joshua git commit: Minor fix in
sevenize_my_conf_plz.
Posted by mj...@apache.org.
Minor fix in sevenize_my_conf_plz.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/39f84855
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/39f84855
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/39f84855
Branch: refs/heads/7_confsystem
Commit: 39f84855d988c3a60f07a79acbe9395e0501cb2b
Parents: 14ede56
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Mon Sep 19 10:19:33 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Mon Sep 19 10:19:33 2016 +0200
----------------------------------------------------------------------
scripts/compat/sevenize_my_conf_plz.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/39f84855/scripts/compat/sevenize_my_conf_plz.py
----------------------------------------------------------------------
diff --git a/scripts/compat/sevenize_my_conf_plz.py b/scripts/compat/sevenize_my_conf_plz.py
index ccdf036..6f2f1b0 100755
--- a/scripts/compat/sevenize_my_conf_plz.py
+++ b/scripts/compat/sevenize_my_conf_plz.py
@@ -93,7 +93,7 @@ for line in sys.stdin:
features.append('class = %s, lm_type = %s, lm_order = %s, lm_file = %s' % (className, lm_type, order, path))
- elif line.startswith('feature-function'):
+ elif line.startswith('feature-function') or line.startswith('feature_function'):
_, feature = re.split(r'\s*=\s*', line, 1)
features.append(parse_args_to_string(feature))
[05/21] incubator-joshua git commit: Fixed two of the three
NumTranslationOptionsTest test cases to work with new configuration. Packed
Grammar test fails, probably missing configuration parameter.
Posted by mj...@apache.org.
Fixed two of the three NumTranslationOptionsTest test cases to work with new configuration. Packed Grammar test fails, probably missing configuration parameter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d117c177
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d117c177
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d117c177
Branch: refs/heads/7_confsystem
Commit: d117c177d9130281f2b057fd1fdaf7de43c4f3b8
Parents: 89a112a
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 18:41:57 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 18:46:02 2016 +0200
----------------------------------------------------------------------
.../NumTranslationOptionsNoDotChartTest.conf | 28 +++++++
.../NumTranslationOptionsNoDotChartTest.gold | 4 +
.../cky/NumTranslationOptionsPackedTest.conf | 27 +++++++
.../cky/NumTranslationOptionsPackedTest.gold | 4 +
.../decoder/cky/NumTranslationOptionsTest.conf | 27 +++++++
.../decoder/cky/NumTranslationOptionsTest.gold | 4 +
.../decoder/cky/NumTranslationOptionsTest.in | 1 +
.../decoder/cky/NumTranslationOptionsTest.java | 83 ++++++--------------
8 files changed, 118 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf
new file mode 100644
index 0000000..a80bd4c
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.conf
@@ -0,0 +1,28 @@
+num_translation_options = 3
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c ||| %s ||| %f
+use_unique_nbest = true
+top_n = 5
+use_dot_chart = false
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/num_translation_options/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = 1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold
new file mode 100644
index 0000000..d130edd
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsNoDotChartTest.gold
@@ -0,0 +1,4 @@
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
new file mode 100644
index 0000000..dc4d6a3
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.conf
@@ -0,0 +1,27 @@
+num_translation_options = 3
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c ||| %s ||| %f
+use_unique_nbest = true
+top_n = 5
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/num_translation_options/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = PackedGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.packed }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = 1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold
new file mode 100644
index 0000000..d9e97f5
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsPackedTest.gold
@@ -0,0 +1,4 @@
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf
new file mode 100644
index 0000000..a8b8723
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.conf
@@ -0,0 +1,27 @@
+num_translation_options = 3
+mark_oovs = false
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+output_format = %c ||| %s ||| %f
+use_unique_nbest = true
+top_n = 5
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/num_translation_options/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = pt, span_limit = 12, path = src/test/resources/decoder/num_translation_options/grammar.gz }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/num_translation_options/glue-grammar }
+]
+
+weights = {
+ OOVPenalty = 1.0
+ WordPenalty = 1
+ glue_0 = -1
+ lm_0 = 1.2373676802179452
+ pt_0 = -1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold
new file mode 100644
index 0000000..d9e97f5
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.gold
@@ -0,0 +1,4 @@
+-19.196 ||| i like taco bell ||| pt_0=-4.000000 lm_0=-17.448898 glue_0=-1.000000 WordPenalty=-2.605767
+-19.733 ||| i love taco bell ||| pt_0=-5.000000 lm_0=-18.690487 glue_0=-1.000000 WordPenalty=-2.605767
+-22.883 ||| i appreciate taco bell ||| pt_0=-3.000000 lm_0=-19.620453 glue_0=-1.000000 WordPenalty=-2.605767
+-424.954 ||| yo quiero taco bell ||| lm_0=-21.293470 glue_0=-4.000000 WordPenalty=-2.605767 OOVPenalty=-400.000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in
new file mode 100644
index 0000000..1ef14e5
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.in
@@ -0,0 +1 @@
+yo quiero taco bell
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d117c177/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
index 7038241..66db0ad 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NumTranslationOptionsTest.java
@@ -18,23 +18,21 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
/**
* Tests that num_translation_options is enforced for hierarchical decoders
*/
public class NumTranslationOptionsTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -45,62 +43,27 @@ public class NumTranslationOptionsTest {
}
}
- @Test
- public void givenInput_whenDecodingWithNumTranslationOptions3_thenScoreAndTranslationCorrect()
- throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/input");
-
- // When
- configureDecoder("src/test/resources/decoder/num_translation_options/joshua.config", true);
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/output.gold");
- assertEquals(decodedStrings, goldStrings);
- }
-
- @Test
- public void givenInput_whenDecodingWithNumTranslationOptions3AndNoDotChart_thenScoreAndTranslationCorrect()
- throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/input");
-
- // When
- configureDecoder("src/test/resources/decoder/num_translation_options/joshua.config", false);
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/output-no-dot-chart.gold");
- assertEquals(decodedStrings, goldStrings);
+ @DataProvider(name = "testFiles")
+ public Object[][] lmFiles() {
+ return new Object[][] {
+ { "NumTranslationOptionsTest.conf", "NumTranslationOptionsTest.in",
+ "NumTranslationOptionsTest.gold" },
+ { "NumTranslationOptionsNoDotChartTest.conf", "NumTranslationOptionsTest.in",
+ "NumTranslationOptionsNoDotChartTest.gold" },
+ { "NumTranslationOptionsPackedTest.conf", "NumTranslationOptionsTest.in",
+ "NumTranslationOptionsPackedTest.gold" } };
}
- @Test
- public void givenInput_whenDecodingWithNumTranslationOptions3AndPacked_thenScoreAndTranslationCorrect()
- throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/input");
+ @Test(dataProvider = "testFiles")
+ public void givenInput_whenDecodingWithNumTranslationOptions_thenScoreAndTranslationCorrect(
+ String confFile, String inFile, String goldFile) throws Exception {
+ String inputPath = this.getClass().getResource(inFile).getFile();
+ String goldPath = this.getClass().getResource(goldFile).getFile();
+ Config config = parseResources(this.getClass(), confFile)
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- // When
- configureDecoder("src/test/resources/decoder/num_translation_options/joshua-packed.config",
- true);
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile(
- "src/test/resources/decoder/num_translation_options/output-packed.gold");
- assertEquals(decodedStrings, goldStrings);
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
- public void configureDecoder(String pathToConfig, boolean useDotChart) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- joshuaConfig.use_dot_chart = useDotChart;
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
}
[03/21] incubator-joshua git commit: Fixed NAryTest to work with new
configuration
Posted by mj...@apache.org.
Fixed NAryTest to work with new configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/28f48103
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/28f48103
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/28f48103
Branch: refs/heads/7_confsystem
Commit: 28f48103f7ea7c9e8ad53b549f376fe38170c576
Parents: c23af30
Author: Michael A. Hedderich <mi...@users.noreply.github.com>
Authored: Sun Sep 18 18:25:00 2016 +0200
Committer: Michael A. Hedderich <mi...@users.noreply.github.com>
Committed: Sun Sep 18 18:25:00 2016 +0200
----------------------------------------------------------------------
.../org/apache/joshua/decoder/cky/NAryTest.conf | 24 ++++++++++
.../org/apache/joshua/decoder/cky/NAryTest.gold | 2 +
.../org/apache/joshua/decoder/cky/NAryTest.in | 2 +
.../org/apache/joshua/decoder/cky/NAryTest.java | 49 ++++----------------
4 files changed, 38 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/28f48103/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.conf b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.conf
new file mode 100644
index 0000000..2fe9529
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.conf
@@ -0,0 +1,24 @@
+mark_oovs = true
+default_non_terminal = X
+goalSymbol = GOAL
+pop_limit = 100
+use_unique_nbest = true
+use_tree_nbest = false
+add_combined_cost = true
+top_n = 1
+output_format = %c %s
+weights_file = src/test/resources/decoder/n-ary/weights
+
+feature_functions = [
+ { class = LanguageModel, lm_type = kenlm, lm_order = 5, lm_file = src/test/resources/decoder/n-ary/lm.gz }
+ { class = WordPenalty }
+ { class = OOVPenalty }
+]
+
+grammars = [
+ { class = TextGrammar, owner = phrase, span_limit = 20, path = src/test/resources/decoder/n-ary/grammar }
+ { class = TextGrammar, owner = glue, span_limit = -1, path = src/test/resources/decoder/n-ary/glue-grammar }
+]
+
+weights = {
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/28f48103/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.gold b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.gold
new file mode 100644
index 0000000..742b98c
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.gold
@@ -0,0 +1,2 @@
+-11.503 Goats eat cheese
+-4.414 i will go home
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/28f48103/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.in
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.in b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.in
new file mode 100644
index 0000000..86437df
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.in
@@ -0,0 +1,2 @@
+CHEESE GOATS EAT
+3 1 4 2
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/28f48103/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.java
index 6a2071a..5c5b4bf 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/NAryTest.java
@@ -18,38 +18,17 @@
*/
package org.apache.joshua.decoder.cky;
-import static org.apache.joshua.decoder.cky.TestUtil.decodeList;
-import static org.apache.joshua.decoder.cky.TestUtil.loadStringsFromFile;
-import static org.testng.Assert.assertEquals;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-import java.util.List;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static org.apache.joshua.decoder.cky.TestUtil.decodeAndAssertDecodedOutputEqualsGold;
import org.apache.joshua.decoder.Decoder;
-import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.util.io.KenLmTestUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.Test;
+import com.typesafe.config.Config;
+
public class NAryTest {
- private JoshuaConfiguration joshuaConfig;
private Decoder decoder;
@AfterMethod
@@ -62,21 +41,13 @@ public class NAryTest {
@Test
public void givenInput_whenNAryDecoding_thenScoreAndTranslationCorrect() throws Exception {
- // Given
- List<String> inputStrings = loadStringsFromFile("src/test/resources/decoder/n-ary/input.txt");
+ String inputPath = this.getClass().getResource("NAryTest.in").getFile();
+ String goldPath = this.getClass().getResource("NAryTest.gold").getFile();
+ Config config = parseResources(this.getClass(), "NAryTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ KenLmTestUtil.Guard(() -> decoder = new Decoder(config));
- // When
- configureDecoder("src/test/resources/decoder/n-ary/joshua.config");
- List<String> decodedStrings = decodeList(inputStrings, decoder, joshuaConfig);
-
- // Then
- List<String> goldStrings = loadStringsFromFile("src/test/resources/decoder/n-ary/output.gold");
- assertEquals(decodedStrings, goldStrings);
+ decodeAndAssertDecodedOutputEqualsGold(inputPath, decoder, goldPath);
}
- public void configureDecoder(String pathToConfig) throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.readConfigFile(pathToConfig);
- KenLmTestUtil.Guard(() -> decoder = new Decoder(joshuaConfig));
- }
}