You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/01 02:51:10 UTC
[15/94] [abbrv] [partial] incubator-joshua git commit: Pulled
JOSHUA-252 changes and Resolved Merge Conflicts
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/segment-oovs/output.expected
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/segment-oovs/output.expected b/src/test/resources/decoder/segment-oovs/output.expected
new file mode 100644
index 0000000..a2786c2
--- /dev/null
+++ b/src/test/resources/decoder/segment-oovs/output.expected
@@ -0,0 +1,82 @@
+0 ||| a g o lf ho le ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 ||| 3.406
+0 ||| a g o lf hole ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 ||| 2.972
+0 ||| a go lf ho le ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 ||| 2.972
+0 ||| a go lf hole ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.606 SourcePath=0.000 ||| 2.537
+0 ||| a golf hole ||| lm_0=-203.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 SourcePath=0.000 ||| 2.103
+0 ||| ein_OOV g o lf ho le ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-100.000 ||| -96.526
+0 ||| a g_OOV o lf ho le ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-100.000 ||| -96.594
+0 ||| a g o lf ho ch_OOV ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-100.000 ||| -96.594
+0 ||| a g o_OOV lf ho le ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-100.000 ||| -96.594
+0 ||| ein_OOV g o lf hole ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -96.960
+0 ||| ein_OOV go lf ho le ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -96.960
+0 ||| a go_OOV lf ho le ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.028
+0 ||| a g_OOV o lf hole ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.028
+0 ||| a go lf ho ch_OOV ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.028
+0 ||| a g o lflo_OOV le ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.028
+0 ||| a g o_OOV lf hole ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.028
+0 ||| ein_OOV go lf hole ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.394
+0 ||| a go_OOV lf hole ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.463
+0 ||| a g o lfloch_OOV ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.463
+0 ||| a go lflo_OOV le ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.463
+0 ||| ein_OOV golf hole ||| lm_0=-302.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.829
+0 ||| a golf_OOV hole ||| lm_0=-203.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.897
+0 ||| a go lfloch_OOV ||| lm_0=-203.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.897
+0 ||| a golf loch_OOV ||| lm_0=-203.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-100.000 ||| -97.897
+0 ||| a golfloch_OOV ||| lm_0=-103.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=2.000 WordPenalty=-1.737 SourcePath=0.000 OOVPenalty=-100.000 ||| -98.331
+0 ||| ein_OOV g o lf ho ch_OOV ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.526
+0 ||| ein_OOV g o_OOV lf ho le ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.526
+0 ||| ein_OOV g_OOV o lf ho le ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.526
+0 ||| a g_OOV o lf ho ch_OOV ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.594
+0 ||| a g_OOV o_OOV lf ho le ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.594
+0 ||| a g o_OOV lf ho ch_OOV ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.594
+0 ||| ein_OOV g o lflo_OOV le ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.960
+0 ||| ein_OOV go lf ho ch_OOV ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.960
+0 ||| ein_OOV go_OOV lf ho le ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.960
+0 ||| ein_OOV g o_OOV lf hole ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.960
+0 ||| ein_OOV g_OOV o lf hole ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -196.960
+0 ||| a g_OOV o lflo_OOV le ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.028
+0 ||| a g o lflo_OOV ch_OOV ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.028
+0 ||| a g o_OOV lflo_OOV le ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.028
+0 ||| a go_OOV lf ho ch_OOV ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.028
+0 ||| a g_OOV o_OOV lf hole ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.028
+0 ||| ein_OOV go_OOV lf hole ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.394
+0 ||| ein_OOV go lflo_OOV le ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.394
+0 ||| ein_OOV g o lfloch_OOV ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.394
+0 ||| a g_OOV o lfloch_OOV ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.463
+0 ||| a go lflo_OOV ch_OOV ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.463
+0 ||| a g o_OOV lfloch_OOV ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.463
+0 ||| a go_OOV lflo_OOV le ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.463
+0 ||| ein_OOV go lfloch_OOV ||| lm_0=-302.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.829
+0 ||| ein_OOV golf loch_OOV ||| lm_0=-302.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.829
+0 ||| ein_OOV golf_OOV hole ||| lm_0=-302.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.829
+0 ||| a golf_OOV loch_OOV ||| lm_0=-203.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.897
+0 ||| a go_OOV lfloch_OOV ||| lm_0=-203.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-200.000 ||| -197.897
+0 ||| ein_OOV golfloch_OOV ||| lm_0=-202.125 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=2.000 WordPenalty=-1.737 SourcePath=0.000 OOVPenalty=-200.000 ||| -198.263
+0 ||| ein_OOV g_OOV o_OOV lf ho le ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.526
+0 ||| ein_OOV g o_OOV lf ho ch_OOV ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.526
+0 ||| ein_OOV g_OOV o lf ho ch_OOV ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.526
+0 ||| a g_OOV o_OOV lf ho ch_OOV ||| lm_0=-503.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.594
+0 ||| ein_OOV g o lflo_OOV ch_OOV ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.960
+0 ||| ein_OOV g_OOV o lflo_OOV le ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.960
+0 ||| ein_OOV g_OOV o_OOV lf hole ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.960
+0 ||| ein_OOV go_OOV lf ho ch_OOV ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.960
+0 ||| ein_OOV g o_OOV lflo_OOV le ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -296.960
+0 ||| a g_OOV o lflo_OOV ch_OOV ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.028
+0 ||| a g_OOV o_OOV lflo_OOV le ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.028
+0 ||| a g o_OOV lflo_OOV ch_OOV ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.028
+0 ||| ein_OOV g_OOV o lfloch_OOV ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.394
+0 ||| ein_OOV go lflo_OOV ch_OOV ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.394
+0 ||| ein_OOV g o_OOV lfloch_OOV ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.394
+0 ||| ein_OOV go_OOV lflo_OOV le ||| lm_0=-402.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.394
+0 ||| a go_OOV lflo_OOV ch_OOV ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.463
+0 ||| a g_OOV o_OOV lfloch_OOV ||| lm_0=-303.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.463
+0 ||| ein_OOV golf_OOV loch_OOV ||| lm_0=-302.125 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.829
+0 ||| ein_OOV go_OOV lfloch_OOV ||| lm_0=-302.125 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=3.000 WordPenalty=-2.171 SourcePath=0.000 OOVPenalty=-300.000 ||| -297.829
+0 ||| ein_OOV g_OOV o_OOV lf ho ch_OOV ||| lm_0=-602.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.474 SourcePath=0.000 OOVPenalty=-400.000 ||| -396.526
+0 ||| ein_OOV g_OOV o_OOV lflo_OOV le ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-400.000 ||| -396.960
+0 ||| ein_OOV g_OOV o lflo_OOV ch_OOV ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-400.000 ||| -396.960
+0 ||| ein_OOV g o_OOV lflo_OOV ch_OOV ||| lm_0=-502.125 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-400.000 ||| -396.960
+0 ||| a g_OOV o_OOV lflo_OOV ch_OOV ||| lm_0=-403.040 tm_pt_0=-0.000 tm_pt_1=-0.228 tm_pt_2=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-400.000 ||| -397.028
+0 ||| ein_OOV go_OOV lflo_OOV ch_OOV ||| lm_0=-402.125 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-400.000 ||| -397.394
+0 ||| ein_OOV g_OOV o_OOV lfloch_OOV ||| lm_0=-402.125 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=4.000 WordPenalty=-2.606 SourcePath=0.000 OOVPenalty=-400.000 ||| -397.394
+0 ||| ein_OOV g_OOV o_OOV lflo_OOV ch_OOV ||| lm_0=-502.125 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=5.000 WordPenalty=-3.040 SourcePath=0.000 OOVPenalty=-500.000 ||| -496.960
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/segment-oovs/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/segment-oovs/test.sh b/src/test/resources/decoder/segment-oovs/test.sh
new file mode 100644
index 0000000..d941de4
--- /dev/null
+++ b/src/test/resources/decoder/segment-oovs/test.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+cat input.txt | $JOSHUA/bin/joshua-decoder -m 500m -c config -maxlen 10 -segment-oovs > output 2> log
+
+if [[ $? -ne 0 ]]; then
+ exit 1
+fi
+
+diff -u output output.expected > diff
+
+if [[ $? -eq 0 ]]; then
+ rm -f output log diff
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/grammar b/src/test/resources/decoder/source-annotations/grammar
new file mode 100644
index 0000000..5afcf6e
--- /dev/null
+++ b/src/test/resources/decoder/source-annotations/grammar
@@ -0,0 +1,5 @@
+[X] ||| mis ||| my ||| 1 ||| 0-0
+[X] ||| amigos ||| friends ||| 1 ||| 0-0
+[X] ||| me ||| me ||| 1 ||| 0-0
+[X] ||| llaman ||| call ||| 1 ||| 0-0
+[X] ||| me llaman ||| call me ||| 1 ||| 0-1 1-0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/grammar.glue
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/grammar.glue b/src/test/resources/decoder/source-annotations/grammar.glue
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/src/test/resources/decoder/source-annotations/grammar.glue
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/input.txt
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/input.txt b/src/test/resources/decoder/source-annotations/input.txt
new file mode 100644
index 0000000..f0a8a76
--- /dev/null
+++ b/src/test/resources/decoder/source-annotations/input.txt
@@ -0,0 +1 @@
+mis[tag=ADJ;num=PL;class=OOV] amigos me llaman
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/joshua.config b/src/test/resources/decoder/source-annotations/joshua.config
new file mode 100644
index 0000000..ffd2f96
--- /dev/null
+++ b/src/test/resources/decoder/source-annotations/joshua.config
@@ -0,0 +1,140 @@
+# This file is a template for the Joshua pipeline; variables enclosed
+# in <angle-brackets> are substituted by the pipeline script as
+# appropriate. This file also serves to document Joshua's many
+# parameters.
+
+# These are the grammar file specifications. Joshua supports an
+# arbitrary number of grammar files, each specified on its own line
+# using the following format:
+#
+# tm = TYPE OWNER LIMIT FILE
+#
+# TYPE is "packed", "thrax", or "samt". The latter denotes the format
+# used in Zollmann and Venugopal's SAMT decoder
+# (http://www.cs.cmu.edu/~zollmann/samt/).
+#
+# OWNER is the "owner" of the rules in the grammar; this is used to
+# determine which set of phrasal features apply to the grammar's
+# rules. Having different owners allows different features to be
+# applied to different grammars, and for grammars to share features
+# across files.
+#
+# LIMIT is the maximum input span permitted for the application of
+# grammar rules found in the grammar file. A value of -1 implies no limit.
+#
+# FILE is the grammar file (or directory when using packed grammars).
+# The file can be compressed with gzip, which is determined by the
+# presence or absence of a ".gz" file extension.
+#
+# By a convention defined by Chiang (2007), the grammars are split
+# into two files: the main translation grammar containing all the
+# learned translation rules, and a glue grammar which supports
+# monotonic concatenation of hierarchical phrases. The glue grammar's
+# main distinction from the regular grammar is that the span limit
+# does not apply to it.
+
+tm = thrax pt 20 grammar
+tm = thrax glue -1 grammar.glue
+
+# This symbol is used over unknown words in the source language
+
+default-non-terminal = X
+
+# This is the goal nonterminal, used to determine when a complete
+# parse is found. It should correspond to the root-level rules in the
+# glue grammar.
+
+goal-symbol = GOAL
+
+# Language model config.
+
+# Multiple language models are supported. For each language model,
+# create a line in the following format,
+#
+# lm = TYPE 5 false false 100 FILE
+#
+# where the six fields correspond to the following values:
+# - LM type: one of "kenlm", "berkeleylm", "javalm" (not recommended), or "none"
+# - LM order: the N of the N-gram language model
+# - whether to use left equivalent state (currently not supported)
+# - whether to use right equivalent state (currently not supported)
+# - the ceiling cost of any n-gram (currently ignored)
+# - LM file: the location of the language model file
+# You also need to add a weight for each language model below.
+
+lm = kenlm 5 false false 100 lm.kenlm
+
+# The suffix _OOV is appended to unknown source-language words if this
+# is set to true.
+
+mark-oovs = true
+
+# The pop-limit for decoding. This determines how many hypotheses are
+# considered over each span of the input.
+
+pop-limit = 100
+
+# How many hypotheses to output
+
+top-n = 1
+
+# Whether those hypotheses should be distinct strings
+
+use-unique-nbest = true
+
+# This is the default format of the ouput printed to STDOUT. The variables that can be
+# substituted are:
+#
+# %i: the sentence number (0-indexed)
+# %s: the translated sentence
+# %t: the derivation tree
+# %f: the feature string
+# %c: the model cost
+
+output-format = %i ||| %s ||| %f ||| %c
+
+# When printing the trees (%t in 'output-format'), this controls whether the alignments
+# are also printed.
+
+include-align-index = false
+
+
+## Feature functions and weights.
+#
+# This is the location of the file containing model weights.
+#
+
+
+# For each langage model line listed above, create a weight in the
+# following format: the keyword "lm", a 0-based index, and the weight.
+# lm_INDEX WEIGHT
+
+lm_0 1.0
+
+# The phrasal weights correspond to weights stored with each of the
+# grammar rules. The format is
+#
+# tm_OWNER_COLUMN WEIGHT
+#
+# where COLUMN denotes the 0-based order of the parameter in the
+# grammar file and WEIGHT is the corresponding weight. In the future,
+# we plan to add a sparse feature representation which will simplify
+# this.
+
+tm_pt_0 0.049141264495762726
+tm_glue_0 0.1663815584150378
+
+# The wordpenalty feature counts the number of words in each hypothesis.
+
+WordPenalty -1.5244636836685694
+
+# This feature counts the number of unknown words in the hypothesis.
+
+OOVPenalty 1
+
+# This feature weights paths through an input lattice. It is only activated
+# when decoding lattices.
+
+# And these are the feature functions to activate.
+feature_function = OOVPenalty
+feature_function = WordPenalty
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/lm.kenlm
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/lm.kenlm b/src/test/resources/decoder/source-annotations/lm.kenlm
new file mode 100644
index 0000000..3931a7f
Binary files /dev/null and b/src/test/resources/decoder/source-annotations/lm.kenlm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/output.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/output.gold b/src/test/resources/decoder/source-annotations/output.gold
new file mode 100644
index 0000000..10125f5
--- /dev/null
+++ b/src/test/resources/decoder/source-annotations/output.gold
@@ -0,0 +1,2 @@
+0 ||| my friends call me ||| tm_pt_0=-3.000 tm_glue_0=3.000 lm_0=-11.974 OOVPenalty=0.000 WordPenalty=-2.606 ||| -7.650
+0 ||| my friends call me ||| tm_pt_0=-3.000 tm_glue_0=3.000 lm_0=-111.513 OOVPenalty=0.000 WordPenalty=-2.606 ||| -107.189
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/source-annotations/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/source-annotations/test.sh b/src/test/resources/decoder/source-annotations/test.sh
new file mode 100755
index 0000000..e352af3
--- /dev/null
+++ b/src/test/resources/decoder/source-annotations/test.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tests the language model code that uses the source-side projection instead of the word itself.
+# When translating a word, if there is a source-side annotation of the label "class", and
+# -source-annotations was added to the invocation, the LM will use that source-side class instead
+# of the translated word.
+
+set -u
+
+cat input.txt | $JOSHUA/bin/joshua-decoder -threads 1 -c joshua.config > output 2> log
+cat input.txt | $JOSHUA/bin/joshua-decoder -threads 1 -c joshua.config -source-annotations >> output 2>> log
+
+# Compare
+diff -u output output.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output log output.scores
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/target-bigram/out.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/target-bigram/out.gold b/src/test/resources/decoder/target-bigram/out.gold
new file mode 100644
index 0000000..8d53a28
--- /dev/null
+++ b/src/test/resources/decoder/target-bigram/out.gold
@@ -0,0 +1,3 @@
+0 ||| this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_this=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_UNK=1.000 TargetBigram_is_UNK=1.000 TargetBigram_this_is=1.000 ||| 0.000
+0 ||| this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_UNK=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_UNK=1.000 TargetBigram_UNK_a=1.000 TargetBigram_a_UNK=1.000 ||| 0.000
+0 ||| this is a test ||| tm_glue_0=4.000 TargetBigram_<s>_UNK=1.000 TargetBigram_UNK_</s>=1.000 TargetBigram_UNK_is=1.000 TargetBigram_a_UNK=1.000 TargetBigram_is_a=1.000 ||| 0.000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/target-bigram/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/target-bigram/test.sh b/src/test/resources/decoder/target-bigram/test.sh
new file mode 100755
index 0000000..10679b7
--- /dev/null
+++ b/src/test/resources/decoder/target-bigram/test.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+(echo "this is a test" | $JOSHUA/bin/joshua-decoder -feature-function "TargetBigram -vocab vocab -top-n 2";
+echo "this is a test" | $JOSHUA/bin/joshua-decoder -feature-function "TargetBigram -vocab vocab -top-n 3 -threshold 20";
+echo "this is a test" | $JOSHUA/bin/joshua-decoder -feature-function "TargetBigram -vocab vocab -threshold 10") 2>log > out
+
+# Compare
+diff -u out out.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff out log
+ exit 0
+else
+ exit 1
+fi
+
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/target-bigram/vocab
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/target-bigram/vocab b/src/test/resources/decoder/target-bigram/vocab
new file mode 100644
index 0000000..bdb1cc4
--- /dev/null
+++ b/src/test/resources/decoder/target-bigram/vocab
@@ -0,0 +1,4 @@
+1 this 1
+2 is 17
+3 a 42
+4 test 9
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/too-long/output.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/too-long/output.gold b/src/test/resources/decoder/too-long/output.gold
new file mode 100644
index 0000000..8773765
--- /dev/null
+++ b/src/test/resources/decoder/too-long/output.gold
@@ -0,0 +1,4 @@
+0 ||| as kingfishers ||| tm_glue_0=2.000 ||| 0.000
+0 ||| dragonflies ||| tm_glue_0=1.000 ||| 0.000
+0 ||| (((as tumbled over rim in roundy wells stones ||| tm_glue_0=8.000 ||| 0.000
+0 ||| ||| ||| 0.000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/too-long/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/too-long/test.sh b/src/test/resources/decoder/too-long/test.sh
new file mode 100755
index 0000000..03b8dbc
--- /dev/null
+++ b/src/test/resources/decoder/too-long/test.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+# Ensures that the decoder trims inputs when and only when it should
+
+(
+echo as kingfishers draw fire | joshua -maxlen 2
+echo dragonflies draw flame | joshua -maxlen 1 -lattice-decoding
+echo "(((as tumbled over rim in roundy wells stones ring" | joshua -maxlen 8
+echo "(((like each tucked string tells" | joshua -maxlen 3 -lattice-decoding
+) > output 2> log
+
+diff -u output output.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f log output diff
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/fragment-map.txt
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/fragment-map.txt b/src/test/resources/decoder/tree-output/fragment-map.txt
new file mode 100644
index 0000000..daed866
--- /dev/null
+++ b/src/test/resources/decoder/tree-output/fragment-map.txt
@@ -0,0 +1,2 @@
+(S (NP (PRP "I")) (VP (VB "am"))) ||| I AM
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/glue-grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/glue-grammar b/src/test/resources/decoder/tree-output/glue-grammar
new file mode 100644
index 0000000..1ec13e2
--- /dev/null
+++ b/src/test/resources/decoder/tree-output/glue-grammar
@@ -0,0 +1,6 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [A,2] ||| [GOAL,1] [A,2] ||| -1
+[GOAL] ||| [GOAL,1] [D,2] ||| [GOAL,1] [D,2] ||| -1
+[GOAL] ||| [GOAL,1] [S,2] ||| [GOAL,1] [S,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
+[GOAL] ||| <s> [NP\DT] </s> ||| <s> [NP\DT,1] </s> ||| -1
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/grammar.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/grammar.gz b/src/test/resources/decoder/tree-output/grammar.gz
new file mode 100644
index 0000000..a538500
Binary files /dev/null and b/src/test/resources/decoder/tree-output/grammar.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/input
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/input b/src/test/resources/decoder/tree-output/input
new file mode 100644
index 0000000..a5a18b4
--- /dev/null
+++ b/src/test/resources/decoder/tree-output/input
@@ -0,0 +1,5 @@
+foo bar baz
+an unparseable sentence
+baz
+yo soy
+purchase xslot
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/joshua.config b/src/test/resources/decoder/tree-output/joshua.config
new file mode 100644
index 0000000..d68192d
--- /dev/null
+++ b/src/test/resources/decoder/tree-output/joshua.config
@@ -0,0 +1,45 @@
+lm = kenlm 5 false false 100 lm.gz
+
+tm = thrax pt 12 grammar.gz
+tm = thrax glue -1 glue-grammar
+
+mark_oovs = false
+
+default-non-terminal = X
+goalSymbol = GOAL
+
+#pruning config
+pop-limit = 100
+
+output-format = %i %t
+
+#nbest config
+use_unique_nbest = true
+top_n = 1
+
+feature-function = WordPenalty
+feature-function = OOVPenalty
+
+
+lm_0 1.2373676802179452
+
+tm_pt_0 -2.4497429277910214
+tm_pt_1 0.7224581556224123
+tm_pt_2 -0.31689069155153504
+tm_pt_3 0.33861043967238036
+tm_pt_4 0.03553113401320236
+tm_pt_5 0.19138972284064748
+tm_pt_6 0.3417994095521415
+tm_pt_7 -0.9936312455671283
+tm_pt_8 0.9070737587091975
+tm_pt_9 0.8202511858619419
+tm_pt_10 0.2593091306160006
+tm_pt_11 0.25597137004462134
+tm_pt_12 0.3538894647790496
+tm_pt_13 -0.36212061186692646
+tm_pt_14 -0.32923261148678096
+tm_pt_15 0.5524863522177359
+tm_pt_16 0.23451595442127693
+tm_glue_0 1
+WordPenalty -3.6942747832593694
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/lm.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/lm.gz b/src/test/resources/decoder/tree-output/lm.gz
new file mode 100644
index 0000000..a26335e
Binary files /dev/null and b/src/test/resources/decoder/tree-output/lm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/output.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/output.gold b/src/test/resources/decoder/tree-output/output.gold
new file mode 100644
index 0000000..68a1610
--- /dev/null
+++ b/src/test/resources/decoder/tree-output/output.gold
@@ -0,0 +1,5 @@
+0 (GOAL{0-5} (GOAL{0-4} (GOAL{0-3} (GOAL{0-1} <s>) (A{1-3} (B{1-2} foo) (C{2-3} bar))) (D{3-4} baz)) </s>)
+1 ()
+2 (GOAL{0-3} (GOAL{0-2} (GOAL{0-1} <s>) (D{1-2} baz)) </s>)
+3 (GOAL{0-4} (GOAL{0-3} (GOAL{0-1} <s>) (S{1-3} I AM)) </s>)
+4 (GOAL{0-4} <s> (NP\DT{1-3} right (NN{2-3} xslot)) </s>)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/decoder/tree-output/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/tree-output/test.sh b/src/test/resources/decoder/tree-output/test.sh
new file mode 100755
index 0000000..ea2d126
--- /dev/null
+++ b/src/test/resources/decoder/tree-output/test.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+cat input | $JOSHUA/bin/joshua-decoder -m 1g -threads 1 -c joshua.config > output 2> log
+
+# Compare
+diff -u output output.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff log output output.scores
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar b/src/test/resources/grammar/sparse-features/grammar
new file mode 100644
index 0000000..20d7927
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/grammar
@@ -0,0 +1 @@
+[X] ||| el chico ||| the boy ||| -1 sparse_test_feature=1 svd=1 the_boy=1
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.glue
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.glue b/src/test/resources/grammar/sparse-features/grammar.glue
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/grammar.glue
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.packed/encoding
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.packed/encoding b/src/test/resources/grammar/sparse-features/grammar.packed/encoding
new file mode 100644
index 0000000..bd24719
Binary files /dev/null and b/src/test/resources/grammar/sparse-features/grammar.packed/encoding differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.features
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.features b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.features
new file mode 100644
index 0000000..681af70
Binary files /dev/null and b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.features differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.source
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.source b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.source
new file mode 100644
index 0000000..d723066
Binary files /dev/null and b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.source differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target
new file mode 100644
index 0000000..bb48631
Binary files /dev/null and b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target.lookup
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target.lookup b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target.lookup
new file mode 100644
index 0000000..24fb4f7
Binary files /dev/null and b/src/test/resources/grammar/sparse-features/grammar.packed/slice_00000.target.lookup differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/grammar.packed/vocabulary
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/grammar.packed/vocabulary b/src/test/resources/grammar/sparse-features/grammar.packed/vocabulary
new file mode 100644
index 0000000..2c93d0f
Binary files /dev/null and b/src/test/resources/grammar/sparse-features/grammar.packed/vocabulary differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/joshua-packed.config
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/joshua-packed.config b/src/test/resources/grammar/sparse-features/joshua-packed.config
new file mode 100644
index 0000000..3b73c24
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/joshua-packed.config
@@ -0,0 +1,12 @@
+tm = thrax pt 20 grammar.packed
+tm = thrax glue -1 grammar.glue
+default-non-terminal = X
+goal-symbol = GOAL
+mark-oovs = false
+pop-limit = 100
+top-n = 1
+use-unique-nbest = true
+output-format = %i ||| %s ||| %f ||| %c
+include-align-index = false
+lm_0 1.0
+tm_pt_0 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/joshua.config b/src/test/resources/grammar/sparse-features/joshua.config
new file mode 100644
index 0000000..91d9c48
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/joshua.config
@@ -0,0 +1,12 @@
+tm = thrax pt 20 grammar
+tm = thrax glue -1 grammar.glue
+default-non-terminal = X
+goal-symbol = GOAL
+mark-oovs = false
+pop-limit = 100
+top-n = 1
+use-unique-nbest = true
+output-format = %i ||| %s ||| %f ||| %c
+include-align-index = false
+lm_0 1.0
+tm_pt_0 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/output.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/output.gold b/src/test/resources/grammar/sparse-features/output.gold
new file mode 100644
index 0000000..7e07c66
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/output.gold
@@ -0,0 +1 @@
+0 ||| the boy ||| tm_pt_0=1.000 tm_glue_0=1.000 sparse_test_feature=1.000 svd=1.000 the_boy=1.000 ||| 1.000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/test-packed.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/test-packed.sh b/src/test/resources/grammar/sparse-features/test-packed.sh
new file mode 100755
index 0000000..04d7bb0
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/test-packed.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+echo el chico | $JOSHUA/bin/decoder -c joshua-packed.config -v 0 > output
+
+# Compare
+diff -u output output.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output
+ exit 0
+else
+ exit 1
+fi
+
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/grammar/sparse-features/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/grammar/sparse-features/test.sh b/src/test/resources/grammar/sparse-features/test.sh
new file mode 100755
index 0000000..93a871c
--- /dev/null
+++ b/src/test/resources/grammar/sparse-features/test.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+echo el chico | $JOSHUA/bin/decoder -c joshua.config -v 0 > output 2> log
+
+# Compare
+diff -u output output.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output log
+ exit 0
+else
+ exit 1
+fi
+
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/joshua/README.broken
----------------------------------------------------------------------
diff --git a/src/test/resources/joshua/README.broken b/src/test/resources/joshua/README.broken
new file mode 100644
index 0000000..eaa429a
--- /dev/null
+++ b/src/test/resources/joshua/README.broken
@@ -0,0 +1 @@
+The tests in and beneath this directory do not currently work (2012-06-18).
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/README
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/README b/src/test/resources/lattice-short/README
new file mode 100644
index 0000000..8e66b25
--- /dev/null
+++ b/src/test/resources/lattice-short/README
@@ -0,0 +1,3 @@
+This test ensures that the distance between nodes is computed correctly. In lattices,
+the span (j - i) no longer contains the distance between two words, since words connected
+by two hops in the lattice could have an arbitrary distance between their node IDs.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/glue-grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/glue-grammar b/src/test/resources/lattice-short/glue-grammar
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/src/test/resources/lattice-short/glue-grammar
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/grammar.test
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/grammar.test b/src/test/resources/lattice-short/grammar.test
new file mode 100644
index 0000000..074f969
--- /dev/null
+++ b/src/test/resources/lattice-short/grammar.test
@@ -0,0 +1,3 @@
+[X] ||| a ||| A ||| 1
+[X] ||| b ||| B ||| 2
+[X] ||| x ||| X ||| 3
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/input
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/input b/src/test/resources/lattice-short/input
new file mode 100644
index 0000000..9d3695b
--- /dev/null
+++ b/src/test/resources/lattice-short/input
@@ -0,0 +1,5 @@
+a
+a x
+b
+b x
+((('a',1,1),('b',1,2),),(('x',1,2),),(('x',1,1),),)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/joshua.config b/src/test/resources/lattice-short/joshua.config
new file mode 100644
index 0000000..0e938a5
--- /dev/null
+++ b/src/test/resources/lattice-short/joshua.config
@@ -0,0 +1,39 @@
+tm = thrax pt 1 grammar.test
+tm = thrax glue -1 glue-grammar
+
+#lm config
+lm = berkeleylm 3 false false 100 test.lm
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit = 100
+
+#nbest config
+use_unique_nbest = true
+include-align-index = false
+top-n = 6
+
+lattice-decoding = true
+
+mark-oovs = false
+
+feature-function = OOVPenalty
+feature-function = WordPenalty
+feature-function = SourcePath
+
+###### model weights
+#lm order weight
+lm_0 1.0
+
+#phrasemodel owner column(0-indexed) weight
+tm_pt_0 1.0
+tm_glue_0 0.0
+
+#wordpenalty weight
+WordPenalty -1.0
+SourcePath 1.0
+
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/output.expected
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/output.expected b/src/test/resources/lattice-short/output.expected
new file mode 100644
index 0000000..a712719
--- /dev/null
+++ b/src/test/resources/lattice-short/output.expected
@@ -0,0 +1,18 @@
+0 ||| A ||| tm_pt_0=-1.000 tm_glue_0=1.000 lm_0=-3.040 OOVPenalty=0.000 WordPenalty=-1.303 SourcePath=0.000 ||| -2.737
+0 ||| a ||| tm_pt_0=0.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=-100.000 WordPenalty=-1.303 SourcePath=0.000 ||| -199.378
+1 ||| A X ||| tm_pt_0=-4.000 tm_glue_0=2.000 lm_0=-101.827 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=0.000 ||| -104.090
+1 ||| A x ||| tm_pt_0=-1.000 tm_glue_0=2.000 lm_0=-101.827 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=0.000 ||| -201.090
+1 ||| a X ||| tm_pt_0=-3.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=0.000 ||| -301.944
+1 ||| a x ||| tm_pt_0=0.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-200.000 WordPenalty=-1.737 SourcePath=0.000 ||| -398.944
+2 ||| B ||| tm_pt_0=-2.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=0.000 WordPenalty=-1.303 SourcePath=0.000 ||| -101.378
+2 ||| b ||| tm_pt_0=0.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=-100.000 WordPenalty=-1.303 SourcePath=0.000 ||| -199.378
+3 ||| B X ||| tm_pt_0=-5.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=0.000 ||| -203.944
+3 ||| B x ||| tm_pt_0=-2.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=0.000 ||| -300.944
+3 ||| b X ||| tm_pt_0=-3.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=0.000 ||| -301.944
+3 ||| b x ||| tm_pt_0=0.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-200.000 WordPenalty=-1.737 SourcePath=0.000 ||| -398.944
+4 ||| A X ||| tm_pt_0=-4.000 tm_glue_0=2.000 lm_0=-101.827 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=2.000 ||| -102.090
+4 ||| A x ||| tm_pt_0=-1.000 tm_glue_0=2.000 lm_0=-101.827 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=2.000 ||| -199.090
+4 ||| B X ||| tm_pt_0=-5.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=2.000 ||| -201.944
+4 ||| B x ||| tm_pt_0=-2.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=2.000 ||| -298.944
+4 ||| a X ||| tm_pt_0=-3.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=2.000 ||| -299.944
+4 ||| b X ||| tm_pt_0=-3.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=2.000 ||| -299.944
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/test.lm
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/test.lm b/src/test/resources/lattice-short/test.lm
new file mode 100644
index 0000000..e903b95
--- /dev/null
+++ b/src/test/resources/lattice-short/test.lm
@@ -0,0 +1,113 @@
+
+\data\
+ngram 1=21
+ngram 2=54
+ngram 3=25
+
+\1-grams:
+-0.6811531 </s>
+-99 <s> -1.444227
+-0.9510576 A -1.212613
+-2.273277 control -0.1995491
+-2.273277 day -0.1995491
+-1.982712 green -0.3496768
+-0.9923518 house -1.131215
+-0.83132 is -0.5989807
+-1.982712 mine -0.2861166
+-1.810166 money -0.5005792
+-1.337818 mouse -0.8089954
+-1.810166 none -0.4110553
+-1.810166 of -0.5350947
+-1.982712 one -0.3727064
+-1.810166 out -0.5952836
+-1.014571 small -0.02323565
+-1.591175 that -0.708865
+-1.591175 the -0.6820672
+-1.088991 there -1.161163
+-1.810166 this -0.4176781
+-1.446277 yellow -0.7813725
+
+\2-grams:
+-1.146128 <s> A
+-1.623249 <s> green
+-1.623249 <s> mine
+-1.623249 <s> money
+-1.322219 <s> none
+-1.322219 <s> one
+-1.146128 <s> out 0.4339377
+-0.9242793 <s> that 0
+-1.021189 <s> the 0.3575113
+-0.4191293 <s> there 0
+-1.146128 <s> this
+-1.361728 A green
+-0.6627578 A house -0.154902
+-0.8846066 A mouse 0.2218488
+-0.4586379 A small -0.771571
+-0.6627578 A yellow -0.2730013
+-0.30103 control </s>
+-0.30103 day </s>
+-0.4771213 green </s>
+-0.4771213 green mouse
+-0.1180993 house </s>
+-0.7201593 house is -0.6755285
+-0.2071255 is A 0.1144242
+-1.462398 is mine
+-1.161368 is money 0.1249388
+-1.462398 is none
+-1.279467 is small -0.2445198
+-0.4771213 mine </s>
+-0.4771213 mine is
+-0.1249387 money </s>
+-0.09691001 mouse </s>
+-1 mouse is
+-0.30103 none </s>
+-0.60206 none is
+-0.60206 of A
+-0.60206 of control
+-0.60206 of the
+-0.4771213 one </s>
+-0.4771213 one day
+-0.1249387 out of
+-1.095823 small </s>
+-1.095823 small house 0.3424227
+-0.8016323 small mouse 0.5308477
+-0.9777236 small yellow
+-0.07918125 that is 0.1991566
+-0.1760913 the house -0.02783953
+-0.7781513 the small
+-0.02632894 there is -0.4820847
+-0.60206 this house
+-0.60206 this is
+-0.60206 this small
+-0.4259687 yellow house 0.455127
+-0.60206 yellow mouse 0.2218488
+-0.60206 yellow small
+
+\3-grams:
+-1.271663 is A house
+-0.9542425 is A mouse
+-0.3521825 is A small
+-0.5563025 is A yellow
+-0.07918125 A house </s>
+-0.3222193 small house </s>
+-0.6184504 the house is
+-0.4935117 yellow house </s>
+-0.09691001 house is small
+-0.39794 that is A
+-0.05799195 there is A
+-0.1760913 is money </s>
+-0.1760913 A mouse </s>
+-0.4935117 small mouse </s>
+-0.1760913 yellow mouse </s>
+-0.4935117 <s> out of
+-0.3521825 A small house
+-0.6532125 A small mouse
+-0.6532125 A small yellow
+-0.3222193 is small </s>
+-0.07918125 <s> that is
+-0.6184504 <s> the house
+-0.02632894 <s> there is
+-0.39794 A yellow house
+-0.39794 A yellow small
+
+\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice-short/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice-short/test.sh b/src/test/resources/lattice-short/test.sh
new file mode 100755
index 0000000..ec63c72
--- /dev/null
+++ b/src/test/resources/lattice-short/test.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+cat input | $JOSHUA/bin/joshua-decoder -m 500m -config joshua.config 2> log > output
+
+if [[ $? -ne 0 ]]; then
+ exit 1
+fi
+
+diff -u output output.expected > diff
+
+if [[ $? -eq 0 ]]; then
+ rm -f output log diff
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/.gitignore
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/.gitignore b/src/test/resources/lattice/.gitignore
new file mode 100644
index 0000000..90080e5
--- /dev/null
+++ b/src/test/resources/lattice/.gitignore
@@ -0,0 +1,3 @@
+log
+test.nbest
+diff
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/README
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/README b/src/test/resources/lattice/README
new file mode 100644
index 0000000..68c88b0
--- /dev/null
+++ b/src/test/resources/lattice/README
@@ -0,0 +1,4 @@
+Tests decoding of lattices (sentence 1) and ensures that lattices are not trimmed
+for length but that sentences are (sentence 2).
+
+Sentence 3 tests scientific notation on arc labels.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/glue-grammar
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/glue-grammar b/src/test/resources/lattice/glue-grammar
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/src/test/resources/lattice/glue-grammar
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/grammar.test
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/grammar.test b/src/test/resources/lattice/grammar.test
new file mode 100644
index 0000000..3618e60
--- /dev/null
+++ b/src/test/resources/lattice/grammar.test
@@ -0,0 +1,204 @@
+[X] ||| das ||| the ||| 0.30103 0.0 0.30103
+[X] ||| das ||| that ||| 0.30103 0.0 0.30103
+[X] ||| das [X,1] ||| that [X,1] ||| 0.25527248 0.0 0.30103
+[X] ||| das [X,1] ||| the [X,1] ||| 0.3521825 0.0 0.30103
+[X] ||| haus ||| house ||| 0.0222764 0.0 0.021189298
+[X] ||| haus ||| small house ||| 1.30103 0.0 1.3434086
+[X] ||| haus [X,1] ||| house [X,1] ||| 0.0 0.0 0.021189298
+[X] ||| klein ||| small ||| 0.0 0.43365556 0.0
+[X] ||| klein [X,1] ||| [X,1] small ||| 0.0 0.43365556 0.0
+[X] ||| ist ||| is ||| 0.0 0.34845465 0.0
+[X] ||| [X,1] haus ||| [X,1] house ||| 0.0 0.0 0.021189298
+[X] ||| [X,1] ist ||| is [X,1] ||| 0.09691001 0.34845465 0.0
+[X] ||| [X,1] ist ||| [X,1] is ||| 0.69897 0.34845465 0.0
+[X] ||| das haus ||| the house ||| 0.0 0.0 0.32221928
+[X] ||| das haus [X,1] ||| the house [X,1] ||| 0.0 0.0 0.32221928
+[X] ||| das [X,1] ist ||| that is [X,1] ||| 0.0 0.34845465 0.30103
+[X] ||| klein ist ||| is small ||| 0.0 0.7821102 0.0
+[X] ||| [X,1] klein ist ||| [X,1] is small ||| 0.0 0.7821102 0.0
+[X] ||| haus klein ist ||| house is small ||| 0.0 0.7821102 0.021189298
+[X] ||| das haus klein ist ||| the house is small ||| 0.0 0.7821102 0.32221928
+[X] ||| es ||| there ||| 0.0 0.0 0.0
+[X] ||| es [X,1] ||| there [X,1] ||| 0.0 0.0 0.0
+[X] ||| ein ||| a ||| 0.0 0.22847936 0.0
+[X] ||| ein [X,1] ||| a [X,1] ||| 0.0 0.22847936 0.0
+[X] ||| gibt ||| is ||| 0.0 0.258278 0.0
+[X] ||| [X,1] gibt ||| is [X,1] ||| 0.0 0.258278 0.0
+[X] ||| es [X,1] gibt ||| there is [X,1] ||| 0.0 0.258278 0.0
+[X] ||| ein haus ||| a house ||| 0.079181254 0.22847936 0.021189298
+[X] ||| ein haus ||| a small house ||| 0.7781512 0.22847936 1.3434086
+[X] ||| ein [X,1] gibt ||| is a [X,1] ||| 0.0 0.48675737 0.0
+[X] ||| [X,1] haus gibt ||| is [X,1] house ||| 0.0 0.258278 0.021189298
+[X] ||| es ein [X,1] gibt ||| there is a [X,1] ||| 0.0 0.48675737 0.0
+[X] ||| es [X,1] haus gibt ||| there is [X,1] house ||| 0.0 0.258278 0.021189298
+[X] ||| ein haus gibt ||| is a small house ||| 0.47712123 0.48675737 1.3434086
+[X] ||| ein haus gibt ||| is a house ||| 0.17609124 0.48675737 0.021189298
+[X] ||| es ein haus gibt ||| there is a small house ||| 0.47712123 0.48675737 1.3434086
+[X] ||| es ein haus gibt ||| there is a house ||| 0.17609124 0.48675737 0.021189298
+[X] ||| klein das [X,1] ||| the [X,1] small ||| 0.0 0.43365556 0.30103
+[X] ||| klein [X,1] ist ||| [X,1] is small ||| 0.0 0.7821102 0.0
+[X] ||| haus ist ||| house is ||| 0.0 0.34845465 0.021189298
+[X] ||| [X,1] haus ist ||| is [X,1] house ||| 0.0 0.34845465 0.021189298
+[X] ||| das haus ist ||| the house is ||| 0.0 0.34845465 0.32221928
+[X] ||| klein das haus ist ||| the house is small ||| 0.0 0.7821102 0.32221928
+[X] ||| dieses ||| this ||| 0.0 0.0 0.0
+[X] ||| dieses [X,1] ||| this [X,1] ||| 0.0 0.0 0.0
+[X] ||| dieses haus ||| this house ||| 0.0 0.0 0.021189298
+[X] ||| dieses haus [X,1] ||| this house [X,1] ||| 0.0 0.0 0.021189298
+[X] ||| dieses [X,1] ist ||| this is [X,1] ||| 0.0 0.34845465 0.0
+[X] ||| dieses haus klein ist ||| this house is small ||| 0.0 0.7821102 0.021189298
+[X] ||| kleine ||| small ||| 0.0 0.50060236 0.0
+[X] ||| kleine [X,1] ||| small [X,1] ||| 0.0 0.50060236 0.0
+[X] ||| [X,1] kleine ||| [X,1] small ||| 0.0 0.50060236 0.0
+[X] ||| dieses kleine ||| this small ||| 0.0 0.50060236 0.0
+[X] ||| kleine haus ||| small house ||| 0.0 0.50060236 0.021189298
+[X] ||| dieses kleine haus ||| this small house ||| 0.0 0.50060236 0.021189298
+[X] ||| kleines ||| small ||| 0.0 0.57978356 0.0
+[X] ||| kleines [X,1] ||| small [X,1] ||| 0.0 0.57978356 0.0
+[X] ||| [X,1] kleines ||| [X,1] small ||| 0.0 0.57978356 0.0
+[X] ||| ein kleines ||| a small ||| 0.0 0.80826294 0.0
+[X] ||| ein kleines [X,1] ||| a small [X,1] ||| 0.0 0.80826294 0.0
+[X] ||| ein [X,1] haus ||| a [X,1] house ||| 0.0 0.22847936 0.021189298
+[X] ||| kleines haus ||| small house ||| 0.0 0.57978356 0.021189298
+[X] ||| [X,1] kleines haus ||| [X,1] small house ||| 0.0 0.57978356 0.021189298
+[X] ||| ein kleines haus ||| a small house ||| 0.0 0.80826294 0.021189298
+[X] ||| ein kleines [X,1] gibt ||| is a small [X,1] ||| 0.0 1.066541 0.0
+[X] ||| ein [X,1] haus gibt ||| is a [X,1] house ||| 0.0 0.48675737 0.021189298
+[X] ||| [X,1] kleines haus gibt ||| is [X,1] small house ||| 0.0 0.8380616 0.021189298
+[X] ||| es ein kleines [X,1] gibt ||| there is a small [X,1] ||| 0.0 1.066541 0.0
+[X] ||| es ein [X,1] haus gibt ||| there is a [X,1] house ||| 0.0 0.48675737 0.021189298
+[X] ||| es [X,1] kleines haus gibt ||| there is [X,1] small house ||| 0.0 0.8380616 0.021189298
+[X] ||| ein kleines haus gibt ||| is a small house ||| 0.0 1.066541 0.021189298
+[X] ||| es ein kleines haus gibt ||| there is a small house ||| 0.0 1.066541 0.021189298
+[X] ||| ein [X,1] ist ||| is a [X,1] ||| 0.0 0.576934 0.0
+[X] ||| das ein [X,1] ist ||| that is a [X,1] ||| 0.0 0.576934 0.30103
+[X] ||| das [X,1] haus ist ||| that is [X,1] house ||| 0.0 0.34845465 0.32221928
+[X] ||| ein kleines haus ist ||| is a small house ||| 0.0 1.1567175 0.021189298
+[X] ||| das ein kleines haus ist ||| that is a small house ||| 0.0 1.1567175 0.32221928
+[X] ||| dieses ein [X,1] ist ||| this is a [X,1] ||| 0.0 0.576934 0.0
+[X] ||| dieses [X,1] haus ist ||| this is [X,1] house ||| 0.0 0.34845465 0.021189298
+[X] ||| dieses ein kleines haus ist ||| this is a small house ||| 0.0 1.1567175 0.021189298
+[X] ||| das kleine ||| the small ||| 0.0 0.50060236 0.30103
+[X] ||| das kleine haus ||| the small house ||| 0.0 0.50060236 0.32221928
+[X] ||| das klein ist ||| that is small ||| 0.0 0.7821102 0.30103
+[X] ||| eines ||| one ||| 0.0 0.30103 0.0
+[X] ||| tages ||| day ||| 0.0 0.0 0.0
+[X] ||| eines tages ||| one day ||| 0.0 0.30103 0.0
+[X] ||| eins ||| one ||| 0.0 0.30103 0.0
+[X] ||| keins ||| none ||| 0.0 0.0 0.0
+[X] ||| keins [X,1] ||| none [X,1] ||| 0.0 0.0 0.0
+[X] ||| aus ||| out ||| 0.47712123 0.0 0.22184873
+[X] ||| aus ||| out of ||| 0.17609124 0.0 0.61978877
+[X] ||| aus [X,1] ||| out [X,1] ||| 0.47712123 0.0 0.22184873
+[X] ||| aus [X,1] ||| out of [X,1] ||| 0.17609124 0.0 0.61978877
+[X] ||| dem haeuschen ||| of control ||| 0.0 0.60206 0.30103
+[X] ||| aus dem haeuschen ||| out of control ||| 0.0 0.60206 0.5228787
+[X] ||| eine ||| a ||| 0.0 0.38818017 0.0
+[X] ||| eine [X,1] ||| a [X,1] ||| 0.0 0.38818017 0.0
+[X] ||| maus ||| mouse ||| 0.0 0.0 0.0
+[X] ||| maus [X,1] ||| mouse [X,1] ||| 0.0 0.0 0.0
+[X] ||| [X,1] maus ||| [X,1] mouse ||| 0.0 0.0 0.0
+[X] ||| eine maus ||| a mouse ||| 0.0 0.38818017 0.0
+[X] ||| eine maus [X,1] ||| a mouse [X,1] ||| 0.0 0.38818017 0.0
+[X] ||| eine [X,1] gibt ||| is a [X,1] ||| 0.0 0.6464582 0.0
+[X] ||| [X,1] maus gibt ||| is [X,1] mouse ||| 0.0 0.258278 0.0
+[X] ||| es eine [X,1] gibt ||| there is a [X,1] ||| 0.0 0.6464582 0.0
+[X] ||| es [X,1] maus gibt ||| there is [X,1] mouse ||| 0.0 0.258278 0.0
+[X] ||| eine maus gibt ||| is a mouse ||| 0.0 0.6464582 0.0
+[X] ||| es eine maus gibt ||| there is a mouse ||| 0.0 0.6464582 0.0
+[X] ||| eine kleine ||| a small ||| 0.0 0.8887825 0.0
+[X] ||| eine kleine [X,1] ||| a small [X,1] ||| 0.0 0.8887825 0.0
+[X] ||| eine [X,1] maus ||| a [X,1] mouse ||| 0.0 0.38818017 0.0
+[X] ||| kleine maus ||| small mouse ||| 0.0 0.50060236 0.0
+[X] ||| [X,1] kleine maus ||| [X,1] small mouse ||| 0.0 0.50060236 0.0
+[X] ||| eine kleine maus ||| a small mouse ||| 0.0 0.8887825 0.0
+[X] ||| eine kleine [X,1] gibt ||| is a small [X,1] ||| 0.0 1.1470605 0.0
+[X] ||| eine [X,1] maus gibt ||| is a [X,1] mouse ||| 0.0 0.6464582 0.0
+[X] ||| [X,1] kleine maus gibt ||| is [X,1] small mouse ||| 0.0 0.7588804 0.0
+[X] ||| es eine kleine [X,1] gibt ||| there is a small [X,1] ||| 0.0 1.1470605 0.0
+[X] ||| es eine [X,1] maus gibt ||| there is a [X,1] mouse ||| 0.0 0.6464582 0.0
+[X] ||| es [X,1] kleine maus gibt ||| there is [X,1] small mouse ||| 0.0 0.7588804 0.0
+[X] ||| eine kleine maus gibt ||| is a small mouse ||| 0.0 1.1470605 0.0
+[X] ||| es eine kleine maus gibt ||| there is a small mouse ||| 0.0 1.1470605 0.0
+[X] ||| keins gibt ||| is none ||| 0.0 0.258278 0.0
+[X] ||| es keins gibt ||| there is none ||| 0.0 0.258278 0.0
+[X] ||| meins ||| mine ||| 0.0 0.0 0.0
+[X] ||| meins [X,1] ||| mine [X,1] ||| 0.0 0.0 0.0
+[X] ||| meins ist ||| is mine ||| 0.0 0.34845465 0.0
+[X] ||| das meins ist ||| that is mine ||| 0.0 0.34845465 0.30103
+[X] ||| meins klein ist ||| mine is small ||| 0.0 0.7821102 0.0
+[X] ||| aus das ||| out of the ||| 0.0 0.0 0.92081875
+[X] ||| aus das haus ||| out of the house ||| 0.0 0.0 0.942008
+[X] ||| aus ein ||| out of a ||| 0.0 0.22847936 0.61978877
+[X] ||| aus ein haus ||| out of a house ||| 0.0 0.22847936 0.64097804
+[X] ||| keins klein ist ||| none is small ||| 0.0 0.7821102 0.0
+[X] ||| maus [X,1] ist ||| mouse is [X,1] ||| 0.0 0.34845465 0.0
+[X] ||| eine maus [X,1] ist ||| a mouse is [X,1] ||| 0.0 0.7366348 0.0
+[X] ||| ein haus ist ||| is a house ||| 0.0 0.576934 0.021189298
+[X] ||| [X,1] ein haus ist ||| [X,1] is a house ||| 0.0 0.576934 0.021189298
+[X] ||| maus ein haus ist ||| mouse is a house ||| 0.0 0.576934 0.021189298
+[X] ||| eine maus ein haus ist ||| a mouse is a house ||| 0.0 0.9651142 0.021189298
+[X] ||| gelb ||| yellow ||| 0.0 0.0 0.0
+[X] ||| gelb [X,1] ||| yellow [X,1] ||| 0.0 0.0 0.0
+[X] ||| [X,1] gelb ||| [X,1] yellow ||| 0.0 0.0 0.0
+[X] ||| eine gelb ||| a yellow ||| 0.0 0.38818017 0.0
+[X] ||| eine gelb [X,1] ||| a yellow [X,1] ||| 0.0 0.38818017 0.0
+[X] ||| gelb maus ||| yellow mouse ||| 0.0 0.0 0.0
+[X] ||| [X,1] gelb maus ||| [X,1] yellow mouse ||| 0.0 0.0 0.0
+[X] ||| eine gelb maus ||| a yellow mouse ||| 0.0 0.38818017 0.0
+[X] ||| eine gelb [X,1] gibt ||| is a yellow [X,1] ||| 0.0 0.6464582 0.0
+[X] ||| [X,1] gelb maus gibt ||| is [X,1] yellow mouse ||| 0.0 0.258278 0.0
+[X] ||| es eine gelb [X,1] gibt ||| there is a yellow [X,1] ||| 0.0 0.6464582 0.0
+[X] ||| es [X,1] gelb maus gibt ||| there is [X,1] yellow mouse ||| 0.0 0.258278 0.0
+[X] ||| eine gelb maus gibt ||| is a yellow mouse ||| 0.0 0.6464582 0.0
+[X] ||| es eine gelb maus gibt ||| there is a yellow mouse ||| 0.0 0.6464582 0.0
+[X] ||| ein gelb ||| a yellow ||| 0.0 0.22847936 0.0
+[X] ||| ein gelb [X,1] ||| a yellow [X,1] ||| 0.0 0.22847936 0.0
+[X] ||| gelb haus ||| yellow house ||| 0.0 0.0 0.021189298
+[X] ||| [X,1] gelb haus ||| [X,1] yellow house ||| 0.0 0.0 0.021189298
+[X] ||| ein gelb haus ||| a yellow house ||| 0.0 0.22847936 0.021189298
+[X] ||| ein gelb [X,1] gibt ||| is a yellow [X,1] ||| 0.0 0.48675737 0.0
+[X] ||| [X,1] gelb haus gibt ||| is [X,1] yellow house ||| 0.0 0.258278 0.021189298
+[X] ||| es ein gelb [X,1] gibt ||| there is a yellow [X,1] ||| 0.0 0.48675737 0.0
+[X] ||| es [X,1] gelb haus gibt ||| there is [X,1] yellow house ||| 0.0 0.258278 0.021189298
+[X] ||| ein gelb haus gibt ||| is a yellow house ||| 0.0 0.48675737 0.021189298
+[X] ||| es ein gelb haus gibt ||| there is a yellow house ||| 0.0 0.48675737 0.021189298
+[X] ||| ein gelb haus ist ||| is a yellow house ||| 0.0 0.576934 0.021189298
+[X] ||| das ein gelb haus ist ||| that is a yellow house ||| 0.0 0.576934 0.32221928
+[X] ||| gelb kleines ||| yellow small ||| 0.0 0.57978356 0.0
+[X] ||| ein gelb kleines ||| a yellow small ||| 0.0 0.80826294 0.0
+[X] ||| gelb kleines haus ||| yellow small house ||| 0.0 0.57978356 0.021189298
+[X] ||| ein gelb kleines haus ||| a yellow small house ||| 0.0 0.80826294 0.021189298
+[X] ||| ein gelb kleines haus gibt ||| is a yellow small house ||| 0.0 1.066541 0.021189298
+[X] ||| kleines gelb ||| small yellow ||| 0.0 0.57978356 0.0
+[X] ||| ein kleines gelb ||| a small yellow ||| 0.0 0.80826294 0.0
+[X] ||| kleines gelb haus ||| small yellow house ||| 0.0 0.57978356 0.021189298
+[X] ||| ein kleines gelb haus ||| a small yellow house ||| 0.0 0.80826294 0.021189298
+[X] ||| ein kleines gelb haus gibt ||| is a small yellow house ||| 0.0 1.066541 0.021189298
+[X] ||| kleine gelb ||| small yellow ||| 0.0 0.50060236 0.0
+[X] ||| eine kleine gelb ||| a small yellow ||| 0.0 0.8887825 0.0
+[X] ||| kleine gelb maus ||| small yellow mouse ||| 0.0 0.50060236 0.0
+[X] ||| eine kleine gelb maus ||| a small yellow mouse ||| 0.0 0.8887825 0.0
+[X] ||| eine kleine gelb maus gibt ||| is a small yellow mouse ||| 0.0 1.1470605 0.0
+[X] ||| gelb kleine ||| yellow small ||| 0.0 0.50060236 0.0
+[X] ||| eine gelb kleine ||| a yellow small ||| 0.0 0.8887825 0.0
+[X] ||| gelb kleine maus ||| yellow small mouse ||| 0.0 0.50060236 0.0
+[X] ||| eine gelb kleine maus ||| a yellow small mouse ||| 0.0 0.8887825 0.0
+[X] ||| eine gelb kleine maus gibt ||| is a yellow small mouse ||| 0.0 1.1470605 0.0
+[X] ||| geld ||| money ||| 0.0 0.0 0.0
+[X] ||| geld gibt ||| is money ||| 0.0 0.258278 0.0
+[X] ||| es geld gibt ||| there is money ||| 0.0 0.258278 0.0
+[X] ||| geld ist ||| is money ||| 0.0 0.34845465 0.0
+[X] ||| das geld ist ||| that is money ||| 0.0 0.34845465 0.30103
+[X] ||| gruen ||| green ||| 0.0 0.0 0.0
+[X] ||| eine gruen ||| a green ||| 0.0 0.38818017 0.0
+[X] ||| gruen maus ||| green mouse ||| 0.0 0.0 0.0
+[X] ||| eine gruen maus ||| a green mouse ||| 0.0 0.38818017 0.0
+[X] ||| golf ||| golf ||| 0 0 0
+[X] ||| loch ||| hole ||| 0 0 0
+[X] ||| g ||| g ||| 0 0 0
+[X] ||| o ||| o ||| 0 0 0
+[X] ||| go ||| go ||| 0 0 0
+[X] ||| lflo ||| lf ho ||| 0 0 0
+[X] ||| ch ||| le ||| 0 0 0
+[X] ||| lfloch ||| lf hole ||| 0 0 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/joshua.config
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/joshua.config b/src/test/resources/lattice/joshua.config
new file mode 100644
index 0000000..e3b24c8
--- /dev/null
+++ b/src/test/resources/lattice/joshua.config
@@ -0,0 +1,47 @@
+tm = thrax pt 1 grammar.test
+tm = thrax glue -1 glue-grammar
+
+#lm config
+lm = berkeleylm 3 false false 100 test.lm
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit = 100
+
+#nbest config
+use_unique_nbest = true
+include-align-index = false
+top_n = 300
+
+mark-oovs = true
+
+# this shouldn't apply to the lattice
+maxlen = 1
+
+lattice-decoding = true
+
+output-format = %i ||| %e ||| %s ||| %f ||| %c
+
+feature-function = OOVPenalty
+feature-function = WordPenalty
+feature-function = SourcePath
+
+###### model weights
+#lm order weight
+lm_0 1.0
+
+#phrasemodel owner column(0-indexed) weight
+tm_pt_0 1.0
+tm_pt_1 0.5
+tm_pt_2 0.5
+
+tm_glue_0 0.0
+
+#wordpenalty weight
+WordPenalty -1.0
+SourcePath 1.0
+
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/output.expected
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/output.expected b/src/test/resources/lattice/output.expected
new file mode 100644
index 0000000..2528f19
--- /dev/null
+++ b/src/test/resources/lattice/output.expected
@@ -0,0 +1,33 @@
+0 ||| dieses haus ||| this house ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-1.866 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -1.140
+0 ||| ein haus ||| a house ||| tm_pt_0=0.000 tm_pt_1=-0.228 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-1.888 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=-0.900 ||| -1.176
+0 ||| haus ||| house ||| tm_pt_0=-0.022 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-2.555 OOVPenalty=0.000 WordPenalty=-1.303 SourcePath=-0.500 ||| -1.785
+0 ||| ein haus ||| a small house ||| tm_pt_0=-0.778 tm_pt_1=-0.228 tm_pt_2=-1.343 tm_glue_0=1.000 lm_0=-2.279 OOVPenalty=0.000 WordPenalty=-2.171 SourcePath=-0.900 ||| -2.572
+0 ||| dieses haus ||| this small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=2.000 lm_0=-3.166 OOVPenalty=0.000 WordPenalty=-2.171 SourcePath=-1.000 ||| -3.967
+0 ||| haus ||| small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=1.000 lm_0=-3.877 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=-0.500 ||| -4.612
+0 ||| haus ||| haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=-100.000 WordPenalty=-1.303 SourcePath=-0.500 ||| -199.878
+0 ||| ein haus ||| ein_OOV house ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-101.110 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-0.900 ||| -200.284
+0 ||| dieses haus ||| dieses_OOV house ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-101.110 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -200.384
+0 ||| dieses haus ||| this haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=2.000 lm_0=-101.827 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -201.090
+0 ||| ein haus ||| a haus_OOV ||| tm_pt_0=0.000 tm_pt_1=-0.228 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-101.827 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-0.900 ||| -201.104
+0 ||| ein haus ||| ein_OOV small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=2.000 lm_0=-102.433 OOVPenalty=-100.000 WordPenalty=-2.171 SourcePath=-0.900 ||| -203.134
+0 ||| dieses haus ||| dieses_OOV small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=2.000 lm_0=-102.433 OOVPenalty=-100.000 WordPenalty=-2.171 SourcePath=-1.000 ||| -203.234
+0 ||| ein haus ||| ein_OOV haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-200.000 WordPenalty=-1.737 SourcePath=-0.900 ||| -399.844
+0 ||| dieses haus ||| dieses_OOV haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-200.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -399.944
+1 ||| ein ||| a ||| tm_pt_0=0.000 tm_pt_1=-0.228 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-3.040 OOVPenalty=0.000 WordPenalty=-1.303 SourcePath=0.000 ||| -1.851
+1 ||| ein ||| ein_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=-100.000 WordPenalty=-1.303 SourcePath=0.000 ||| -199.378
+2 ||| ein haus ||| a house ||| tm_pt_0=0.000 tm_pt_1=-0.228 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-1.888 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=-0.700 ||| -0.976
+2 ||| dieses haus ||| this house ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-1.866 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -1.140
+2 ||| haus ||| house ||| tm_pt_0=-0.022 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-2.555 OOVPenalty=0.000 WordPenalty=-1.303 SourcePath=-0.500 ||| -1.785
+2 ||| ein haus ||| a small house ||| tm_pt_0=-0.778 tm_pt_1=-0.228 tm_pt_2=-1.343 tm_glue_0=1.000 lm_0=-2.279 OOVPenalty=0.000 WordPenalty=-2.171 SourcePath=-0.700 ||| -2.372
+2 ||| dieses haus ||| this small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=2.000 lm_0=-3.166 OOVPenalty=0.000 WordPenalty=-2.171 SourcePath=-1.000 ||| -3.967
+2 ||| haus ||| small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=1.000 lm_0=-3.877 OOVPenalty=0.000 WordPenalty=-1.737 SourcePath=-0.500 ||| -4.612
+2 ||| haus ||| haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=-100.000 WordPenalty=-1.303 SourcePath=-0.500 ||| -199.878
+2 ||| ein haus ||| ein_OOV house ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-101.110 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-0.700 ||| -200.084
+2 ||| dieses haus ||| dieses_OOV house ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-0.021 tm_glue_0=1.000 lm_0=-101.110 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -200.384
+2 ||| ein haus ||| a haus_OOV ||| tm_pt_0=0.000 tm_pt_1=-0.228 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-101.827 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-0.700 ||| -200.904
+2 ||| dieses haus ||| this haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-101.827 OOVPenalty=-100.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -201.090
+2 ||| ein haus ||| ein_OOV small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=2.000 lm_0=-102.433 OOVPenalty=-100.000 WordPenalty=-2.171 SourcePath=-0.700 ||| -202.934
+2 ||| dieses haus ||| dieses_OOV small house ||| tm_pt_0=-1.301 tm_pt_1=0.000 tm_pt_2=-1.343 tm_glue_0=2.000 lm_0=-102.433 OOVPenalty=-100.000 WordPenalty=-2.171 SourcePath=-1.000 ||| -203.234
+2 ||| ein haus ||| ein_OOV haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-200.000 WordPenalty=-1.737 SourcePath=-0.700 ||| -399.644
+2 ||| dieses haus ||| dieses_OOV haus_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=2.000 lm_0=-200.681 OOVPenalty=-200.000 WordPenalty=-1.737 SourcePath=-1.000 ||| -399.944
+3 ||| s� ||| s�_OOV ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_glue_0=1.000 lm_0=-100.681 OOVPenalty=-100.000 WordPenalty=-1.303 SourcePath=0.000 ||| -199.378
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/test-lattice.pdf
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/test-lattice.pdf b/src/test/resources/lattice/test-lattice.pdf
new file mode 100644
index 0000000..2070e6b
Binary files /dev/null and b/src/test/resources/lattice/test-lattice.pdf differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/test.lm
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/test.lm b/src/test/resources/lattice/test.lm
new file mode 100644
index 0000000..b64b6fc
--- /dev/null
+++ b/src/test/resources/lattice/test.lm
@@ -0,0 +1,113 @@
+
+\data\
+ngram 1=21
+ngram 2=54
+ngram 3=25
+
+\1-grams:
+-0.6811531 </s>
+-99 <s> -1.444227
+-0.9510576 a -1.212613
+-2.273277 control -0.1995491
+-2.273277 day -0.1995491
+-1.982712 green -0.3496768
+-0.9923518 house -1.131215
+-0.83132 is -0.5989807
+-1.982712 mine -0.2861166
+-1.810166 money -0.5005792
+-1.337818 mouse -0.8089954
+-1.810166 none -0.4110553
+-1.810166 of -0.5350947
+-1.982712 one -0.3727064
+-1.810166 out -0.5952836
+-1.014571 small -0.02323565
+-1.591175 that -0.708865
+-1.591175 the -0.6820672
+-1.088991 there -1.161163
+-1.810166 this -0.4176781
+-1.446277 yellow -0.7813725
+
+\2-grams:
+-1.146128 <s> a
+-1.623249 <s> green
+-1.623249 <s> mine
+-1.623249 <s> money
+-1.322219 <s> none
+-1.322219 <s> one
+-1.146128 <s> out 0.4339377
+-0.9242793 <s> that 0
+-1.021189 <s> the 0.3575113
+-0.4191293 <s> there 0
+-1.146128 <s> this
+-1.361728 a green
+-0.6627578 a house -0.154902
+-0.8846066 a mouse 0.2218488
+-0.4586379 a small -0.771571
+-0.6627578 a yellow -0.2730013
+-0.30103 control </s>
+-0.30103 day </s>
+-0.4771213 green </s>
+-0.4771213 green mouse
+-0.1180993 house </s>
+-0.7201593 house is -0.6755285
+-0.2071255 is a 0.1144242
+-1.462398 is mine
+-1.161368 is money 0.1249388
+-1.462398 is none
+-1.279467 is small -0.2445198
+-0.4771213 mine </s>
+-0.4771213 mine is
+-0.1249387 money </s>
+-0.09691001 mouse </s>
+-1 mouse is
+-0.30103 none </s>
+-0.60206 none is
+-0.60206 of a
+-0.60206 of control
+-0.60206 of the
+-0.4771213 one </s>
+-0.4771213 one day
+-0.1249387 out of
+-1.095823 small </s>
+-1.095823 small house 0.3424227
+-0.8016323 small mouse 0.5308477
+-0.9777236 small yellow
+-0.07918125 that is 0.1991566
+-0.1760913 the house -0.02783953
+-0.7781513 the small
+-0.02632894 there is -0.4820847
+-0.60206 this house
+-0.60206 this is
+-0.60206 this small
+-0.4259687 yellow house 0.455127
+-0.60206 yellow mouse 0.2218488
+-0.60206 yellow small
+
+\3-grams:
+-1.271663 is a house
+-0.9542425 is a mouse
+-0.3521825 is a small
+-0.5563025 is a yellow
+-0.07918125 a house </s>
+-0.3222193 small house </s>
+-0.6184504 the house is
+-0.4935117 yellow house </s>
+-0.09691001 house is small
+-0.39794 that is a
+-0.05799195 there is a
+-0.1760913 is money </s>
+-0.1760913 a mouse </s>
+-0.4935117 small mouse </s>
+-0.1760913 yellow mouse </s>
+-0.4935117 <s> out of
+-0.3521825 a small house
+-0.6532125 a small mouse
+-0.6532125 a small yellow
+-0.3222193 is small </s>
+-0.07918125 <s> that is
+-0.6184504 <s> the house
+-0.02632894 <s> there is
+-0.39794 a yellow house
+-0.39794 a yellow small
+
+\end\
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/test.plf
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/test.plf b/src/test/resources/lattice/test.plf
new file mode 100644
index 0000000..632e952
--- /dev/null
+++ b/src/test/resources/lattice/test.plf
@@ -0,0 +1,4 @@
+((('ein',-0.2,1),('dieses',-0.3,1),('haus',-0.5,2),),(('haus',-0.7,1),),)
+ein haus
+((('ein',1.56462193e-07,1),('dieses',-0.3,1),('haus',-0.5,2),),(('haus',-0.7,1),),)
+((('s�', 0, 1),),)
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lattice/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/lattice/test.sh b/src/test/resources/lattice/test.sh
new file mode 100755
index 0000000..6e2caba
--- /dev/null
+++ b/src/test/resources/lattice/test.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# The number of MB to give to Java's heap
+# For this example 500 is minimum
+# For 32-bit Java 2048 (or so) is the maximum
+
+rm -f test.nbest test.1best
+
+cat test.plf | $JOSHUA/bin/joshua-decoder -m 500m -c joshua.config > output 2> log
+
+if [[ $? -ne 0 ]]; then
+ exit 1
+fi
+
+diff -u output output.expected > diff
+
+if [[ $? -eq 0 ]]; then
+ rm -f output log diff
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lm/berkeley/lm
----------------------------------------------------------------------
diff --git a/src/test/resources/lm/berkeley/lm b/src/test/resources/lm/berkeley/lm
new file mode 100644
index 0000000..05b4e6b
--- /dev/null
+++ b/src/test/resources/lm/berkeley/lm
@@ -0,0 +1,16 @@
+
+\data\
+ngram 1=5
+ngram 2=3
+
+\1-grams:
+-99.000000 <unk>
+-99.000000 <s> -1.752754
+-2.034158 the -0.800943
+-5.318589 chat-rooms -0.151088
+-1.495702 </s>
+
+\2-grams:
+-1.773970 <s> the
+-4.878868 the chat-rooms
+-0.499794 chat-rooms </s>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lm/berkeley/lm.berkeleylm
----------------------------------------------------------------------
diff --git a/src/test/resources/lm/berkeley/lm.berkeleylm b/src/test/resources/lm/berkeley/lm.berkeleylm
new file mode 100644
index 0000000..c048464
Binary files /dev/null and b/src/test/resources/lm/berkeley/lm.berkeleylm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lm/berkeley/lm.berkeleylm.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/lm/berkeley/lm.berkeleylm.gz b/src/test/resources/lm/berkeley/lm.berkeleylm.gz
new file mode 100644
index 0000000..f9f8d16
Binary files /dev/null and b/src/test/resources/lm/berkeley/lm.berkeleylm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lm/berkeley/lm.gz
----------------------------------------------------------------------
diff --git a/src/test/resources/lm/berkeley/lm.gz b/src/test/resources/lm/berkeley/lm.gz
new file mode 100644
index 0000000..ae47266
Binary files /dev/null and b/src/test/resources/lm/berkeley/lm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ef91969a/src/test/resources/lm/berkeley/output.gold
----------------------------------------------------------------------
diff --git a/src/test/resources/lm/berkeley/output.gold b/src/test/resources/lm/berkeley/output.gold
new file mode 100644
index 0000000..c0d30a4
--- /dev/null
+++ b/src/test/resources/lm/berkeley/output.gold
@@ -0,0 +1,4 @@
+tm_glue_0=2.000 lm_0=-7.153
+tm_glue_0=2.000 lm_0=-7.153
+tm_glue_0=2.000 lm_0=-7.153
+tm_glue_0=2.000 lm_0=-7.153