You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/08/23 22:18:07 UTC
[50/50] [abbrv] incubator-joshua git commit: moved resources files
moved resources files
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/19b557bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/19b557bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/19b557bc
Branch: refs/heads/7
Commit: 19b557bca85b0ccb0009f0c6865d39ba438d507a
Parents: dc75670
Author: Matt Post <po...@cs.jhu.edu>
Authored: Tue Aug 23 17:17:04 2016 -0500
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Tue Aug 23 17:17:04 2016 -0500
----------------------------------------------------------------------
.../decoder/ff/lm/class_lm/ClassMapTest.java | 2 +-
joshua-core/src/test/resources/berkeley_lm/lm | 16 +
.../test/resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
.../test/resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
.../src/test/resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
joshua-core/src/test/resources/grammar.glue | 4 +
.../resources/kbest_extraction/glue-grammar | 3 +
.../src/test/resources/kbest_extraction/grammar | 25 +
.../resources/kbest_extraction/joshua.config | 27 +
.../src/test/resources/kbest_extraction/lm.gz | Bin 0 -> 2466496 bytes
.../test/resources/kbest_extraction/output.gold | 3126 ++++++++++++++++++
.../kbest_extraction/output.scores.gold | 3126 ++++++++++++++++++
.../src/test/resources/kenlm/oilers.kenlm | Bin 0 -> 49011 bytes
.../src/test/resources/lm_oov/joshua.config | 17 +
.../src/test/resources/phrase_decoder/config | 29 +
.../resources/phrase_decoder/constrained.config | 28 +
.../phrase_decoder/constrained.output.gold | 5 +
.../src/test/resources/phrase_decoder/lm.1.gz | Bin 0 -> 2235 bytes
.../test/resources/phrase_decoder/output.gold | 1 +
.../test/resources/phrase_decoder/rules.1.gz | Bin 0 -> 2998042 bytes
joshua-core/src/test/resources/wa_grammar | 3 +
.../src/test/resources/wa_grammar.packed/config | 2 +
.../test/resources/wa_grammar.packed/encoding | Bin 0 -> 154 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 0 -> 45 bytes
.../wa_grammar.packed/slice_00000.features | Bin 0 -> 47 bytes
.../wa_grammar.packed/slice_00000.source | Bin 0 -> 204 bytes
.../wa_grammar.packed/slice_00000.target | Bin 0 -> 128 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../test/resources/wa_grammar.packed/vocabulary | Bin 0 -> 238 bytes
src/test/resources/berkeley_lm/lm | 16 -
src/test/resources/berkeley_lm/lm.berkeleylm | Bin 4294 -> 0 bytes
src/test/resources/berkeley_lm/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
src/test/resources/berkeley_lm/lm.gz | Bin 162 -> 0 bytes
src/test/resources/grammar.glue | 4 -
.../resources/kbest_extraction/glue-grammar | 3 -
src/test/resources/kbest_extraction/grammar | 25 -
.../resources/kbest_extraction/joshua.config | 27 -
src/test/resources/kbest_extraction/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/kbest_extraction/output.gold | 3126 ------------------
.../kbest_extraction/output.scores.gold | 3126 ------------------
src/test/resources/kenlm/oilers.kenlm | Bin 49011 -> 0 bytes
src/test/resources/lm_oov/joshua.config | 17 -
src/test/resources/phrase_decoder/config | 29 -
.../resources/phrase_decoder/constrained.config | 28 -
.../phrase_decoder/constrained.output.gold | 5 -
src/test/resources/phrase_decoder/lm.1.gz | Bin 2235 -> 0 bytes
src/test/resources/phrase_decoder/output.gold | 1 -
src/test/resources/phrase_decoder/rules.1.gz | Bin 2998042 -> 0 bytes
src/test/resources/wa_grammar | 3 -
src/test/resources/wa_grammar.packed/config | 2 -
src/test/resources/wa_grammar.packed/encoding | Bin 154 -> 0 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 45 -> 0 bytes
.../wa_grammar.packed/slice_00000.features | Bin 47 -> 0 bytes
.../wa_grammar.packed/slice_00000.source | Bin 204 -> 0 bytes
.../wa_grammar.packed/slice_00000.target | Bin 128 -> 0 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
src/test/resources/wa_grammar.packed/vocabulary | Bin 238 -> 0 bytes
57 files changed, 6413 insertions(+), 6413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassMapTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassMapTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassMapTest.java
index 5d37a05..15096f1 100644
--- a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassMapTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/class_lm/ClassMapTest.java
@@ -45,7 +45,7 @@ public class ClassMapTest {
@Test
public void givenClassMapFile_whenClassMapRead_thenEntriesAreRead() {
// GIVEN
- final String classMapFile = "./src/test/resources/lm/class_lm/class.map";
+ final String classMapFile = "src/test/resources/lm/class_lm/class.map";
// WHEN
final ClassMap classMap = new ClassMap(classMapFile);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/berkeley_lm/lm
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/berkeley_lm/lm b/joshua-core/src/test/resources/berkeley_lm/lm
new file mode 100644
index 0000000..05b4e6b
--- /dev/null
+++ b/joshua-core/src/test/resources/berkeley_lm/lm
@@ -0,0 +1,16 @@
+
+\data\
+ngram 1=5
+ngram 2=3
+
+\1-grams:
+-99.000000 <unk>
+-99.000000 <s> -1.752754
+-2.034158 the -0.800943
+-5.318589 chat-rooms -0.151088
+-1.495702 </s>
+
+\2-grams:
+-1.773970 <s> the
+-4.878868 the chat-rooms
+-0.499794 chat-rooms </s>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm b/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm
new file mode 100644
index 0000000..c048464
Binary files /dev/null and b/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm.gz b/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm.gz
new file mode 100644
index 0000000..f9f8d16
Binary files /dev/null and b/joshua-core/src/test/resources/berkeley_lm/lm.berkeleylm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/berkeley_lm/lm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/berkeley_lm/lm.gz b/joshua-core/src/test/resources/berkeley_lm/lm.gz
new file mode 100644
index 0000000..ae47266
Binary files /dev/null and b/joshua-core/src/test/resources/berkeley_lm/lm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/grammar.glue
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/grammar.glue b/joshua-core/src/test/resources/grammar.glue
new file mode 100644
index 0000000..69e1520
--- /dev/null
+++ b/joshua-core/src/test/resources/grammar.glue
@@ -0,0 +1,4 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
+[GOAL] ||| <s> [X,1] </s> ||| <s> [X,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/kbest_extraction/glue-grammar
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/kbest_extraction/glue-grammar b/joshua-core/src/test/resources/kbest_extraction/glue-grammar
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/joshua-core/src/test/resources/kbest_extraction/glue-grammar
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/kbest_extraction/grammar
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/kbest_extraction/grammar b/joshua-core/src/test/resources/kbest_extraction/grammar
new file mode 100644
index 0000000..a03b2d9
--- /dev/null
+++ b/joshua-core/src/test/resources/kbest_extraction/grammar
@@ -0,0 +1,25 @@
+[X] ||| a ||| A ||| 2
+[X] ||| a ||| B ||| 3
+[X] ||| a ||| C ||| 5
+[X] ||| a ||| D ||| 7
+[X] ||| a ||| E ||| 11
+[X] ||| b ||| A ||| 13
+[X] ||| b ||| B ||| 17
+[X] ||| b ||| C ||| 19
+[X] ||| b ||| D ||| 23
+[X] ||| b ||| E ||| 29
+[X] ||| c ||| A ||| 31
+[X] ||| c ||| B ||| 37
+[X] ||| c ||| C ||| 41
+[X] ||| c ||| D ||| 43
+[X] ||| c ||| E ||| 47
+[X] ||| d ||| A ||| 53
+[X] ||| d ||| B ||| 59
+[X] ||| d ||| C ||| 61
+[X] ||| d ||| D ||| 67
+[X] ||| d ||| E ||| 71
+[X] ||| e ||| A ||| 73
+[X] ||| e ||| B ||| 79
+[X] ||| e ||| C ||| 83
+[X] ||| e ||| D ||| 89
+[X] ||| e ||| E ||| 97
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/kbest_extraction/joshua.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/kbest_extraction/joshua.config b/joshua-core/src/test/resources/kbest_extraction/joshua.config
new file mode 100644
index 0000000..69ec9c9
--- /dev/null
+++ b/joshua-core/src/test/resources/kbest_extraction/joshua.config
@@ -0,0 +1,27 @@
+feature-function = StateMinimizingLanguageModel -lm_type kenlm -lm_order 5 -lm_file src/test/resources/kbest_extraction/lm.gz
+
+tm = thrax -owner pt -maxspan 12 -path src/test/resources/kbest_extraction/grammar
+tm = thrax -owner glue -maxspan -1 -path src/test/resources/kbest_extraction/glue-grammar
+
+mark_oovs=false
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit=100
+
+#nbest config
+use_unique_nbest=true
+top-n = 3126
+
+#feature_function = WordPenalty
+feature_function = OOVPenalty
+
+# Model Weights ####
+
+lm_0 1
+tm_pt_0 1
+tm_glue_0 1
+OOVPenalty 10000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/19b557bc/joshua-core/src/test/resources/kbest_extraction/lm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/kbest_extraction/lm.gz b/joshua-core/src/test/resources/kbest_extraction/lm.gz
new file mode 100644
index 0000000..a26335e
Binary files /dev/null and b/joshua-core/src/test/resources/kbest_extraction/lm.gz differ