You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/17 12:28:02 UTC
[13/14] incubator-joshua git commit: More Unit test conversions.
Started to load configs as resources from the classpath
More Unit test conversions. Started to load configs as resources from the classpath
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/6f3e3c4f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/6f3e3c4f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/6f3e3c4f
Branch: refs/heads/7_confsystem
Commit: 6f3e3c4f198a5bc59bae995dd97d2342b34eb67c
Parents: b8256c7
Author: Hieber, Felix <fh...@amazon.de>
Authored: Fri Sep 16 15:41:55 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Fri Sep 16 15:41:55 2016 +0200
----------------------------------------------------------------------
.../org/apache/joshua/decoder/cky/TestUtil.java | 1 -
.../kbest_extraction/KBestExtractionTest.java | 1 -
.../joshua/system/StructuredOutputTest.conf | 24 ++++
.../joshua/system/StructuredOutputTest.java | 16 +--
.../system/StructuredTranslationTest.conf | 23 ++++
.../system/StructuredTranslationTest.java | 113 +++++++++++--------
6 files changed, 118 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6f3e3c4f/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
index 02c79c9..640150e 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/cky/TestUtil.java
@@ -81,5 +81,4 @@ public class TestUtil {
final Sentence sentence = new Sentence(input, 0, decoder.getFlags());
return decoder.decode(sentence).toString();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6f3e3c4f/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
index b84d4b3..4a76922 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
@@ -66,7 +66,6 @@ public class KBestExtractionTest {
public void givenInput_whenKbestExtraction_thenOutputIsAsExpected() throws IOException {
final String translation = decode(INPUT).toString();
final String gold = new String(readAllBytes(GOLD_PATH), UTF_8);
- Files.writeFile(translation, new File("resources/kbest_extraction/output.actual"));
assertEquals(translation, gold);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6f3e3c4f/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.conf b/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.conf
new file mode 100644
index 0000000..a5156a6
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.conf
@@ -0,0 +1,24 @@
+top_n = 0
+use_unique_nbest = false
+output_format = %s | %a
+
+
+feature_functions = [
+ {class=OOVPenalty}
+]
+
+grammars=[
+ {class=TextGrammar, owner=pt, span_limit=20, path=src/test/resources/wa_grammar}
+ {class=TextGrammar, owner=glue, span_limit=-1, path=src/test/resources/grammar.glue}
+]
+
+weights = {
+ pt_0=-1
+ pt_1=-1
+ pt_2=-1
+ pt_3=-1
+ pt_4=-1
+ pt_5=-1
+ glue_0=-1
+ OOVPenalty=2
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6f3e3c4f/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java b/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
index d5504f5..908c7ea 100644
--- a/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.joshua.system;
-import static com.typesafe.config.ConfigFactory.parseString;
+import static com.typesafe.config.ConfigFactory.parseResources;
import static com.typesafe.config.ConfigValueFactory.fromAnyRef;
import java.util.Arrays;
@@ -58,18 +58,8 @@ public class StructuredOutputTest {
@BeforeMethod
public void setUp() throws Exception {
- Config weights = parseString(
- "weights = {pt_0=-1, pt_1=-1, pt_2=-1, pt_3=-1, pt_4=-1, pt_5=-1, glue_0=-1, OOVPenalty=2}");
- Config features = parseString("feature_functions = [{class=OOVPenalty}]");
- Config grammars = parseString("grammars=[{class=TextGrammar, owner=pt, span_limit=20, path=src/test/resources/wa_grammar},"
- + "{class=TextGrammar, owner=glue, span_limit=-1, path=src/test/resources/grammar.glue}]");
- Config flags = weights
- .withFallback(features)
- .withFallback(grammars)
- .withFallback(Decoder.getDefaultFlags())
- .withValue("top_n", fromAnyRef(0))
- .withValue("use_unique_nbest", fromAnyRef(false))
- .withValue("output_format", fromAnyRef("%s | %a"));
+ Config flags = parseResources(this.getClass(), "StructuredOutputTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
decoder = new Decoder(flags);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6f3e3c4f/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.conf
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.conf b/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.conf
new file mode 100644
index 0000000..5a9acd3
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.conf
@@ -0,0 +1,23 @@
+top_n = 0
+use_unique_nbest = false
+
+
+feature_functions = [
+ {class=OOVPenalty}
+]
+
+grammars=[
+ {class=TextGrammar, owner=pt, span_limit=20, path=src/test/resources/wa_grammar}
+ {class=TextGrammar, owner=glue, span_limit=-1, path=src/test/resources/grammar.glue}
+]
+
+weights = {
+ pt_0=-1
+ pt_1=-1
+ pt_2=-1
+ pt_3=-1
+ pt_4=-1
+ pt_5=-1
+ glue_0=-1
+ OOVPenalty=1
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/6f3e3c4f/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java b/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
index 5d122e5..6b8530c 100644
--- a/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
@@ -18,6 +18,8 @@
*/
package org.apache.joshua.system;
+import static com.typesafe.config.ConfigFactory.parseResources;
+import static com.typesafe.config.ConfigValueFactory.fromAnyRef;
import static java.util.Arrays.asList;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@@ -73,27 +75,9 @@ public class StructuredTranslationTest {
@BeforeMethod
public void setUp() throws Exception {
- joshuaConfig = new JoshuaConfiguration();
- joshuaConfig.search_algorithm = "cky";
- joshuaConfig.mark_oovs = false;
- joshuaConfig.pop_limit = 100;
- joshuaConfig.use_unique_nbest = false;
- joshuaConfig.include_align_index = false;
- joshuaConfig.topN = 0;
- joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path src/test/resources/wa_grammar");
- joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path src/test/resources/grammar.glue");
- joshuaConfig.goal_symbol = "[GOAL]";
- joshuaConfig.default_non_terminal = "[X]";
- joshuaConfig.features.add("OOVPenalty");
- joshuaConfig.weights.add("pt_0 -1");
- joshuaConfig.weights.add("pt_1 -1");
- joshuaConfig.weights.add("pt_2 -1");
- joshuaConfig.weights.add("pt_3 -1");
- joshuaConfig.weights.add("pt_4 -1");
- joshuaConfig.weights.add("pt_5 -1");
- joshuaConfig.weights.add("glue_0 -1");
- joshuaConfig.weights.add("OOVPenalty 1");
- decoder = new Decoder(joshuaConfig);
+ Config flags = parseResources(this.getClass(), "StructuredTranslationTest.conf")
+ .withFallback(Decoder.getDefaultFlags());
+ decoder = new Decoder(flags);
}
@AfterMethod
@@ -110,11 +94,15 @@ public class StructuredTranslationTest {
@Test
public void givenInput_whenRegularOutputFormat_thenExpectedOutput() {
// GIVEN
- joshuaConfig.use_structured_output = false;
- joshuaConfig.outputFormat = "%s | %a ";
+ boolean useStructuredOutput = false;
+ String outputFormat = "%s | %a ";
// WHEN
- final String translation = decode(INPUT).toString().trim();
+ final String translation = decode(
+ INPUT,
+ decoder.getFlags()
+ .withValue("output_format", fromAnyRef(outputFormat))
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput))).toString().trim();
// THEN
assertEquals(translation, EXPECTED_TRANSLATION + " | " + EXPECTED_WORD_ALIGNMENT_STRING);
@@ -123,12 +111,17 @@ public class StructuredTranslationTest {
@Test
public void givenInput_whenRegularOutputFormatWithTopN1_thenExpectedOutput() {
// GIVEN
- joshuaConfig.use_structured_output = false;
- joshuaConfig.outputFormat = "%s | %e | %a | %c";
- joshuaConfig.topN = 1;
+ boolean useStructuredOutput = false;
+ String outputFormat = "%s | %e | %a | %c";
+ int topN = 1;
// WHEN
- final String translation = decode(INPUT).toString().trim();
+ final String translation = decode(
+ INPUT,
+ decoder.getFlags()
+ .withValue("top_n", fromAnyRef(topN))
+ .withValue("output_format", fromAnyRef(outputFormat))
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput))).toString().trim();
// THEN
assertEquals(translation,
@@ -138,11 +131,16 @@ public class StructuredTranslationTest {
@Test
public void givenInput_whenStructuredOutputFormatWithTopN0_thenExpectedOutput() {
// GIVEN
- joshuaConfig.use_structured_output = true;
- joshuaConfig.topN = 0;
+ boolean useStructuredOutput = true;
+ int topN = 0;
// WHEN
- final Translation translation = decode(INPUT);
+ final Translation translation = decode(
+ INPUT,
+ decoder.getFlags()
+ .withValue("top_n", fromAnyRef(topN))
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput)));
+
final StructuredTranslation structuredTranslation = translation.getStructuredTranslations().get(0);
final String translationString = structuredTranslation.getTranslationString();
final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
@@ -163,11 +161,16 @@ public class StructuredTranslationTest {
@Test
public void givenInput_whenStructuredOutputFormatWithTopN1_thenExpectedOutput() {
// GIVEN
- joshuaConfig.use_structured_output = true;
- joshuaConfig.topN = 1;
+ boolean useStructuredOutput = true;
+ int topN = 0;
// WHEN
- final Translation translation = decode(INPUT);
+ final Translation translation = decode(
+ INPUT,
+ decoder.getFlags()
+ .withValue("top_n", fromAnyRef(topN))
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput)));
+
final List<StructuredTranslation> structuredTranslations = translation.getStructuredTranslations();
final StructuredTranslation structuredTranslation = structuredTranslations.get(0);
final String translationString = structuredTranslation.getTranslationString();
@@ -189,11 +192,16 @@ public class StructuredTranslationTest {
@Test
public void givenInput_whenStructuredOutputFormatWithKBest_thenExpectedOutput() {
// GIVEN
- joshuaConfig.use_structured_output = true;
- joshuaConfig.topN = 100;
+ boolean useStructuredOutput = true;
+ int topN = 100;
// WHEN
- final Translation translation = decode(INPUT);
+ final Translation translation = decode(
+ INPUT,
+ decoder.getFlags()
+ .withValue("top_n", fromAnyRef(topN))
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput)));
+
final List<StructuredTranslation> structuredTranslations = translation.getStructuredTranslations();
final StructuredTranslation viterbiTranslation = structuredTranslations.get(0);
final StructuredTranslation lastKBest = structuredTranslations.get(structuredTranslations.size() - 1);
@@ -215,10 +223,13 @@ public class StructuredTranslationTest {
@Test
public void givenEmptyInput_whenStructuredOutputFormat_thenEmptyOutput() {
// GIVEN
- joshuaConfig.use_structured_output = true;
+ boolean useStructuredOutput = true;
// WHEN
- final Translation translation = decode("");
+ final Translation translation = decode(
+ "",
+ decoder.getFlags()
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput)));
final StructuredTranslation structuredTranslation = translation.getStructuredTranslations().get(0);
final String translationString = structuredTranslation.getTranslationString();
final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
@@ -235,32 +246,44 @@ public class StructuredTranslationTest {
@Test
public void givenOOVInput_whenStructuredOutputFormat_thenOOVOutput() {
// GIVEN
- joshuaConfig.use_structured_output = true;
+ boolean useStructuredOutput = true;
final String input = "gabarbl";
// WHEN
- final Translation translation = decode(input);
+ final Translation translation = decode(
+ input,
+ decoder.getFlags()
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput)));
+
final StructuredTranslation structuredTranslation = translation.getStructuredTranslations().get(0);
final String translationString = structuredTranslation.getTranslationString();
final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
+ final Map<String,Float> translationFeatures = structuredTranslation.getTranslationFeatures();
final float translationScore = structuredTranslation.getTranslationScore();
final List<List<Integer>> wordAlignment = structuredTranslation.getTranslationWordAlignments();
+
+ System.out.println(translationFeatures);
+ System.out.println(decoder.getWeights().textFormat());
// THEN
assertEquals(input, translationString);
assertTrue(translatedTokens.contains(input));
- assertEquals(-99.0, translationScore, 0.00001);
+ assertEquals(translationScore, -99.0, 0.00001);
assertTrue(wordAlignment.contains(asList(0)));
}
@Test
public void givenEmptyInput_whenRegularOutputFormat_thenNewlineOutput() {
// GIVEN
- joshuaConfig.use_structured_output = false;
- joshuaConfig.outputFormat = "%s";
+ boolean useStructuredOutput = false;
+ String outputFormat = "%s";
// WHEN
- final Translation translation = decode("");
+ final Translation translation = decode(
+ "",
+ decoder.getFlags()
+ .withValue("output_format", fromAnyRef(outputFormat))
+ .withValue("use_structured_output", fromAnyRef(useStructuredOutput)));
final String translationString = translation.toString();
// THEN