You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/08/23 03:09:52 UTC
[05/12] incubator-joshua git commit: Moved test file locations from
resources/ to src/test/resources
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/lm_oov/joshua.config
----------------------------------------------------------------------
diff --git a/resources/lm_oov/joshua.config b/resources/lm_oov/joshua.config
deleted file mode 100644
index 3ffb611..0000000
--- a/resources/lm_oov/joshua.config
+++ /dev/null
@@ -1,17 +0,0 @@
-feature-function = LanguageModel -lm_type berkeleylm -lm_order 5 -lm_file resources/berkeley_lm/lm -oov_feature
-
-tm = thrax -owner pt -maxspan 12 -path resources/kbest_extraction/grammar
-tm = thrax -owner glue -maxspan -1 -path resources/kbest_extraction/glue-grammar
-
-top-n = 0
-
-#feature_function = WordPenalty
-feature_function = OOVPenalty
-
-# Model Weights ####
-
-lm_0 0
-lm_0_oov 1
-OOVPenalty 1
-tm_pt_0 0
-tm_glue 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/phrase_decoder/config
----------------------------------------------------------------------
diff --git a/resources/phrase_decoder/config b/resources/phrase_decoder/config
deleted file mode 100644
index de781e3..0000000
--- a/resources/phrase_decoder/config
+++ /dev/null
@@ -1,29 +0,0 @@
-tm = moses -owner pt -maxspan 0 -path resources/phrase_decoder/rules.1.gz -max-source-len 5
-feature-function = StateMinimizingLanguageModel -lm_order 5 -lm_file resources/phrase_decoder/lm.1.gz
-
-search = stack
-
-mark-oovs = false
-pop-limit = 10
-top-n = 1
-
-output-format = %i ||| %s ||| %f ||| %c
-
-include-align-index = true
-reordering-limit = 6
-
-# And these are the feature functions to activate.
-feature-function = OOVPenalty
-feature-function = WordPenalty
-feature-function = Distortion
-feature-function = PhrasePenalty -owner pt
-
-OOVPenalty 1.0
-Distortion 0.114849
-WordPenalty -0.201544
-PhrasePenalty -0.236965
-tm_pt_0 0.0370068
-tm_pt_1 0.0495759
-tm_pt_2 0.196742
-tm_pt_3 0.0745423
-lm_0 0.204412452147565
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/phrase_decoder/constrained.config
----------------------------------------------------------------------
diff --git a/resources/phrase_decoder/constrained.config b/resources/phrase_decoder/constrained.config
deleted file mode 100644
index 4642650..0000000
--- a/resources/phrase_decoder/constrained.config
+++ /dev/null
@@ -1,28 +0,0 @@
-tm = moses pt 0 resources/phrase_decoder/rules.1.gz
-
-lm = kenlm 5 true false 100 resources/phrase_decoder/lm.1.gz
-
-mark-oovs = false
-pop-limit = 10
-top-n = 5
-
-output-format = %i ||| %s ||| %f ||| %c
-
-include-align-index = true
-reordering-limit = 10
-
-# And these are the feature functions to activate.
-feature-function = OOVPenalty
-feature-function = WordPenalty
-feature-function = Distortion
-feature-function = PhrasePenalty -owner pt
-
-OOVPenalty 1.0
-Distortion 0.114849
-WordPenalty -0.201544
-PhrasePenalty -0.236965
-tm_pt_0 0.0370068
-tm_pt_1 0.0495759
-tm_pt_2 0.196742
-tm_pt_3 0.0745423
-lm_0 0.204412452147565
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/phrase_decoder/constrained.output.gold
----------------------------------------------------------------------
diff --git a/resources/phrase_decoder/constrained.output.gold b/resources/phrase_decoder/constrained.output.gold
deleted file mode 100644
index 238387c..0000000
--- a/resources/phrase_decoder/constrained.output.gold
+++ /dev/null
@@ -1,5 +0,0 @@
-0 ||| President Obama |8-8| to |7-7| hinder |4-4| a strategy |0-1| for |3-3| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-15.792 tm_pt_1=-17.550 tm_pt_2=-14.599 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=7.000 ||| -15.163
-0 ||| President Obama |8-8| to |7-7| hinder |4-4| a |0-0| strategy |1-1| for |3-3| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.919 tm_pt_1=-17.550 tm_pt_2=-14.917 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=8.000 ||| -15.505
-0 ||| President Obama |8-8| to hinder |3-4| a strategy |0-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-14.986 tm_pt_1=-17.951 tm_pt_2=-14.075 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=6.000 ||| -15.762
-0 ||| President Obama |8-8| to hinder |3-4| a |0-0| strategy |1-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.112 tm_pt_1=-17.951 tm_pt_2=-14.393 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.103
-0 ||| President Obama |8-8| to |3-3| hinder |4-4| a strategy |0-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.329 tm_pt_1=-17.951 tm_pt_2=-15.136 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.257
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/phrase_decoder/lm.1.gz
----------------------------------------------------------------------
diff --git a/resources/phrase_decoder/lm.1.gz b/resources/phrase_decoder/lm.1.gz
deleted file mode 100644
index 3f4c453..0000000
Binary files a/resources/phrase_decoder/lm.1.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/phrase_decoder/output.gold
----------------------------------------------------------------------
diff --git a/resources/phrase_decoder/output.gold b/resources/phrase_decoder/output.gold
deleted file mode 100644
index 509a3de..0000000
--- a/resources/phrase_decoder/output.gold
+++ /dev/null
@@ -1 +0,0 @@
-0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/phrase_decoder/rules.1.gz
----------------------------------------------------------------------
diff --git a/resources/phrase_decoder/rules.1.gz b/resources/phrase_decoder/rules.1.gz
deleted file mode 100644
index 14466e9..0000000
Binary files a/resources/phrase_decoder/rules.1.gz and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar
----------------------------------------------------------------------
diff --git a/resources/wa_grammar b/resources/wa_grammar
deleted file mode 100644
index 82d0052..0000000
--- a/resources/wa_grammar
+++ /dev/null
@@ -1,3 +0,0 @@
-[X] ||| A [X,1] B1 [X,2] B2 C ||| a b [X,2] c1 [X,1] c2 ||| 1 1 1 1 1 1 OOV=1 ||| 0-0 2-1 4-1 5-3 5-5
-[X] ||| U Z1 Z2 ||| n1 u z ||| 1 1 1 1 1 1 OOV=2 ||| 0-1 1-2 2-2
-[X] ||| K ||| k1 k2 k3 n1 n2 n3 ||| 1 1 1 1 1 1 OOV=4 ||| 0-0 0-1 0-2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/config
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/config b/resources/wa_grammar.packed/config
deleted file mode 100644
index fbc07d0..0000000
--- a/resources/wa_grammar.packed/config
+++ /dev/null
@@ -1,2 +0,0 @@
-max-source-len = 6
-version = 3
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/encoding
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/encoding b/resources/wa_grammar.packed/encoding
deleted file mode 100644
index 630f69f..0000000
Binary files a/resources/wa_grammar.packed/encoding and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/slice_00000.alignments
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/slice_00000.alignments b/resources/wa_grammar.packed/slice_00000.alignments
deleted file mode 100644
index f1425eb..0000000
Binary files a/resources/wa_grammar.packed/slice_00000.alignments and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/slice_00000.features
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/slice_00000.features b/resources/wa_grammar.packed/slice_00000.features
deleted file mode 100644
index 5a4c774..0000000
Binary files a/resources/wa_grammar.packed/slice_00000.features and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/slice_00000.source
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/slice_00000.source b/resources/wa_grammar.packed/slice_00000.source
deleted file mode 100644
index 4607b89..0000000
Binary files a/resources/wa_grammar.packed/slice_00000.source and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/slice_00000.target
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/slice_00000.target b/resources/wa_grammar.packed/slice_00000.target
deleted file mode 100644
index fe11a38..0000000
Binary files a/resources/wa_grammar.packed/slice_00000.target and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/slice_00000.target.lookup
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/slice_00000.target.lookup b/resources/wa_grammar.packed/slice_00000.target.lookup
deleted file mode 100644
index 7d82179..0000000
Binary files a/resources/wa_grammar.packed/slice_00000.target.lookup and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/resources/wa_grammar.packed/vocabulary
----------------------------------------------------------------------
diff --git a/resources/wa_grammar.packed/vocabulary b/resources/wa_grammar.packed/vocabulary
deleted file mode 100644
index 637651e..0000000
Binary files a/resources/wa_grammar.packed/vocabulary and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
index 7752785..cbe6a7f 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
@@ -33,7 +33,7 @@ public class LMBerkeleySentenceProbablityTest {
@Test
public void verifySentenceLogProbability() {
- LMGrammarBerkeley grammar = new LMGrammarBerkeley(2, "resources/berkeley_lm/lm");
+ LMGrammarBerkeley grammar = new LMGrammarBerkeley(2, "src/test/resources/berkeley_lm/lm");
grammar.registerWord("the", 2);
grammar.registerWord("chat-rooms", 3);
grammar.registerWord("<unk>", 0);
@@ -52,7 +52,7 @@ public class LMBerkeleySentenceProbablityTest {
@Test
public void givenUnknownWord_whenIsOov_thenCorrectlyDetected() {
- LMGrammarBerkeley lm = new LMGrammarBerkeley(2, "resources/berkeley_lm/lm");
+ LMGrammarBerkeley lm = new LMGrammarBerkeley(2, "src/test/resources/berkeley_lm/lm");
assertTrue(lm.isOov(Vocabulary.id("UNKNOWN_WORD")));
assertFalse(lm.isOov(Vocabulary.id("chat-rooms")));
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
index b0612d4..32c0762 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
@@ -44,10 +44,10 @@ public class LMGrammarBerkeleyTest {
@DataProvider(name = "languageModelFiles")
public Object[][] lmFiles() {
- return new Object[][]{{"resources/berkeley_lm/lm"},
- {"resources/berkeley_lm/lm.gz"},
- {"resources/berkeley_lm/lm.berkeleylm"},
- {"resources/berkeley_lm/lm.berkeleylm.gz"}};
+ return new Object[][]{{"src/test/resources/berkeley_lm/lm"},
+ {"src/test/resources/berkeley_lm/lm.gz"},
+ {"src/test/resources/berkeley_lm/lm.berkeleylm"},
+ {"src/test/resources/berkeley_lm/lm.berkeleylm.gz"}};
}
@AfterMethod
@@ -74,7 +74,7 @@ public class LMGrammarBerkeleyTest {
public void givenLmWithOovFeature_whenDecoder_thenCorrectFeaturesReturned() {
joshuaConfig = new JoshuaConfiguration();
joshuaConfig.processCommandLineOptions(OPTIONS);
- joshuaConfig.features.add("LanguageModel -lm_type berkeleylm -oov_feature -lm_order 2 -lm_file resources/berkeley_lm/lm");
+ joshuaConfig.features.add("LanguageModel -lm_type berkeleylm -oov_feature -lm_order 2 -lm_file src/test/resources/berkeley_lm/lm");
decoder = new Decoder(joshuaConfig, null);
final String translation = decode(INPUT).toString();
assertEquals(Decoder.weights.getDenseFeatures().size(), 3);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java b/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
index 5946abd..98a94f0 100644
--- a/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
+++ b/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassBasedLanguageModelTest.java
@@ -49,8 +49,8 @@ public class ClassBasedLanguageModelTest {
FeatureVector weights = new FeatureVector();
weights.set("lm_0", WEIGHT);
String[] args = { "-lm_type", "kenlm", "-lm_order", "9",
- "-lm_file", "./src/test/resources/lm/class_lm/class_lm_9gram.gz",
- "-class_map", "./src/test/resources/lm/class_lm/class.map" };
+ "-lm_file", "src/test/resources/lm/class_lm/class_lm_9gram.gz",
+ "-class_map", "src/test/resources/lm/class_lm/class.map" };
JoshuaConfiguration config = new JoshuaConfiguration();
KenLmTestUtil.Guard(() -> ff = new LanguageModelFF(weights, args, config));
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java b/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
index 5b9db06..bc16a40 100644
--- a/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
+++ b/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
@@ -43,9 +43,9 @@ import static org.testng.Assert.assertEquals;
public class KBestExtractionTest {
- private static final String CONFIG = "resources/kbest_extraction/joshua.config";
+ private static final String CONFIG = "src/test/resources/kbest_extraction/joshua.config";
private static final String INPUT = "a b c d e";
- private static final Path GOLD_PATH = Paths.get("resources/kbest_extraction/output.scores.gold");
+ private static final Path GOLD_PATH = Paths.get("src/test/resources/kbest_extraction/output.scores.gold");
private JoshuaConfiguration joshuaConfig = null;
private Decoder decoder = null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java b/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
index 8be3c36..625fe0c 100644
--- a/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
+++ b/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
@@ -36,7 +36,7 @@ import org.testng.annotations.Test;
*/
public class PhraseDecodingTest {
- private static final String CONFIG = "resources/phrase_decoder/config";
+ private static final String CONFIG = "src/test/resources/phrase_decoder/config";
private static final String INPUT = "una estrategia republicana para obstaculizar la reelecci�n de Obama";
private static final String OUTPUT = "0 ||| a strategy republican to hinder reelection Obama ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496";
private static final String OUTPUT_WITH_ALIGNMENTS = "0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496";
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/system/LmOovFeatureTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/LmOovFeatureTest.java b/src/test/java/org/apache/joshua/system/LmOovFeatureTest.java
index d6fc16d..69e5aa9 100644
--- a/src/test/java/org/apache/joshua/system/LmOovFeatureTest.java
+++ b/src/test/java/org/apache/joshua/system/LmOovFeatureTest.java
@@ -32,7 +32,7 @@ import org.testng.annotations.Test;
public class LmOovFeatureTest {
- private static final String CONFIG = "resources/lm_oov/joshua.config";
+ private static final String CONFIG = "src/test/resources/lm_oov/joshua.config";
private static final String INPUT = "a chat-rooms full";
// expecting 2 lm oovs ('a' & 'full') and 2 grammar OOVs ('chat-rooms' & 'full') and score -198.000
private static final String EXPECTED_FEATURES = "tm_pt_0=-2.000 tm_glue_0=3.000 lm_0=-206.718 lm_0_oov=2.000 OOVPenalty=-200.000 | -198.000";
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
index 092dbc6..7b1c47f 100644
--- a/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
+++ b/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
@@ -60,8 +60,8 @@ public class MultithreadedTranslationTests {
joshuaConfig.use_unique_nbest = false;
joshuaConfig.include_align_index = false;
joshuaConfig.topN = 0;
- joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path resources/wa_grammar.packed");
- joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path resources/grammar.glue");
+ joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path src/test/resources/wa_grammar.packed");
+ joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path src/test/resources/grammar.glue");
joshuaConfig.goal_symbol = "[GOAL]";
joshuaConfig.default_non_terminal = "[X]";
joshuaConfig.features.add("OOVPenalty");
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/StructuredOutputTest.java b/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
index 2c9e0bd..051c163 100644
--- a/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
+++ b/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
@@ -62,8 +62,8 @@ public class StructuredOutputTest {
joshuaConfig.use_unique_nbest = false;
joshuaConfig.include_align_index = false;
joshuaConfig.topN = 0;
- joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path resources/wa_grammar");
- joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path resources/grammar.glue");
+ joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path src/test/resources/wa_grammar");
+ joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path src/test/resources/grammar.glue");
joshuaConfig.goal_symbol = "[GOAL]";
joshuaConfig.default_non_terminal = "[X]";
joshuaConfig.features.add("OOVPenalty");
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java b/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
index 4977d08..de95ded 100644
--- a/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
+++ b/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
@@ -81,8 +81,8 @@ public class StructuredTranslationTest {
joshuaConfig.use_unique_nbest = false;
joshuaConfig.include_align_index = false;
joshuaConfig.topN = 0;
- joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path resources/wa_grammar");
- joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path resources/grammar.glue");
+ joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path src/test/resources/wa_grammar");
+ joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path src/test/resources/grammar.glue");
joshuaConfig.goal_symbol = "[GOAL]";
joshuaConfig.default_non_terminal = "[X]";
joshuaConfig.features.add("OOVPenalty");
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/berkeley_lm/lm
----------------------------------------------------------------------
diff --git a/src/test/resources/berkeley_lm/lm b/src/test/resources/berkeley_lm/lm
new file mode 100644
index 0000000..05b4e6b
--- /dev/null
+++ b/src/test/resources/berkeley_lm/lm
@@ -0,0 +1,16 @@
+
+\data\
+ngram 1=5
+ngram 2=3
+
+\1-grams:
+-99.000000 <unk>
+-99.000000 <s> -1.752754
+-2.034158 the -0.800943
+-5.318589 chat-rooms -0.151088
+-1.495702 </s>
+
+\2-grams:
+-1.773970 <s> the
+-4.878868 the chat-rooms
+-0.499794 chat-rooms </s>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/berkeley_lm/lm.berkeleylm
----------------------------------------------------------------------
diff --git a/src/test/resources/berkeley_lm/lm.berkeleylm b/src/test/resources/berkeley_lm/lm.berkeleylm
new file mode 100644
index 0000000..c048464
Binary files /dev/null and b/src/test/resources/berkeley_lm/lm.berkeleylm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/berkeley_lm/lm.berkeleylm.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/berkeley_lm/lm.berkeleylm.gz b/src/test/resources/berkeley_lm/lm.berkeleylm.gz
new file mode 100644
index 0000000..f9f8d16
Binary files /dev/null and b/src/test/resources/berkeley_lm/lm.berkeleylm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/berkeley_lm/lm.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/berkeley_lm/lm.gz b/src/test/resources/berkeley_lm/lm.gz
new file mode 100644
index 0000000..ae47266
Binary files /dev/null and b/src/test/resources/berkeley_lm/lm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/grammar.glue
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar.glue b/src/test/resources/grammar.glue
new file mode 100644
index 0000000..69e1520
--- /dev/null
+++ b/src/test/resources/grammar.glue
@@ -0,0 +1,4 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
+[GOAL] ||| <s> [X,1] </s> ||| <s> [X,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/kbest_extraction/glue-grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/kbest_extraction/glue-grammar b/src/test/resources/kbest_extraction/glue-grammar
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/src/test/resources/kbest_extraction/glue-grammar
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/kbest_extraction/grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/kbest_extraction/grammar b/src/test/resources/kbest_extraction/grammar
new file mode 100644
index 0000000..a03b2d9
--- /dev/null
+++ b/src/test/resources/kbest_extraction/grammar
@@ -0,0 +1,25 @@
+[X] ||| a ||| A ||| 2
+[X] ||| a ||| B ||| 3
+[X] ||| a ||| C ||| 5
+[X] ||| a ||| D ||| 7
+[X] ||| a ||| E ||| 11
+[X] ||| b ||| A ||| 13
+[X] ||| b ||| B ||| 17
+[X] ||| b ||| C ||| 19
+[X] ||| b ||| D ||| 23
+[X] ||| b ||| E ||| 29
+[X] ||| c ||| A ||| 31
+[X] ||| c ||| B ||| 37
+[X] ||| c ||| C ||| 41
+[X] ||| c ||| D ||| 43
+[X] ||| c ||| E ||| 47
+[X] ||| d ||| A ||| 53
+[X] ||| d ||| B ||| 59
+[X] ||| d ||| C ||| 61
+[X] ||| d ||| D ||| 67
+[X] ||| d ||| E ||| 71
+[X] ||| e ||| A ||| 73
+[X] ||| e ||| B ||| 79
+[X] ||| e ||| C ||| 83
+[X] ||| e ||| D ||| 89
+[X] ||| e ||| E ||| 97
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/kbest_extraction/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/kbest_extraction/joshua.config b/src/test/resources/kbest_extraction/joshua.config
new file mode 100644
index 0000000..69ec9c9
--- /dev/null
+++ b/src/test/resources/kbest_extraction/joshua.config
@@ -0,0 +1,27 @@
+feature-function = StateMinimizingLanguageModel -lm_type kenlm -lm_order 5 -lm_file src/test/resources/kbest_extraction/lm.gz
+
+tm = thrax -owner pt -maxspan 12 -path src/test/resources/kbest_extraction/grammar
+tm = thrax -owner glue -maxspan -1 -path src/test/resources/kbest_extraction/glue-grammar
+
+mark_oovs=false
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit=100
+
+#nbest config
+use_unique_nbest=true
+top-n = 3126
+
+#feature_function = WordPenalty
+feature_function = OOVPenalty
+
+# Model Weights ####
+
+lm_0 1
+tm_pt_0 1
+tm_glue_0 1
+OOVPenalty 10000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/20e6bf4b/src/test/resources/kbest_extraction/lm.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/kbest_extraction/lm.gz b/src/test/resources/kbest_extraction/lm.gz
new file mode 100644
index 0000000..a26335e
Binary files /dev/null and b/src/test/resources/kbest_extraction/lm.gz differ